From 0bce4950bf06c5caeea3dd4040e809cb69e96a0f Mon Sep 17 00:00:00 2001 From: Milosz Wasilewski Date: Mon, 24 Aug 2026 16:08:58 -0400 Subject: [PATCH 1/7] gstreamer: bound hardware probes so a wedged codec cannot hang the run The GStreamer video suites can hang indefinitely when the hardware video codec wedges, turning a single failed test case into a dead test run. Observed on hamoa/x1e80100 (LAVA jobs 374345 and 377799), where the VPU throws a system error on first use: qcom-iris aa00000.video-codec: received system error of type 0x5000003 after which every V4L2 access blocks in the kernel. Two unbounded call sites turn that into a full lava-test-shell timeout: - has_element() runs gst-inspect-1.0 with no time bound at all. It is the first thing run_encode_test() calls, and gst-inspect opens every /dev/video* node while validating the plugin registry, so it never returns. In job 374345 the run stopped there, before the pipeline was even built, and burned the entire 10 minute budget. - gstreamer_run_gstlaunch_timeout() uses `timeout --signal=INT` with no --kill-after. A gst-launch-1.0 blocked in a V4L2 ioctl never reacts to SIGINT, so timeout waits for it forever and the nominal duration + 10 bound never takes effect. This is what happened in job 377799. In both cases the remaining test definitions never execute and LAVA reports them as "missing expected test case" failures, which hides the real result: one codec broke, the rest were never tried. Add gstreamer_run_bounded(), which always escalates to SIGKILL after GST_KILL_GRACE seconds, and use it for gst-inspect-1.0, gst-discoverer-1.0 and gst-launch-1.0. A wedged codec now surfaces as an unavailable element or a failed pipeline, and the following test cases still get to run. SIGINT is still the first signal sent to gst-launch-1.0, so muxers keep finalising their output via -e on a healthy timeout; only a process that ignores it is escalated. Behaviour is unchanged when the hardware works. Signed-off-by: Milosz Wasilewski --- Runner/utils/lib_gstreamer.sh | 196 +++++++++++++++++++++++++++++++--- 1 file changed, 184 insertions(+), 12 deletions(-) diff --git a/Runner/utils/lib_gstreamer.sh b/Runner/utils/lib_gstreamer.sh index d852bbf4..e67ac54c 100755 --- a/Runner/utils/lib_gstreamer.sh +++ b/Runner/utils/lib_gstreamer.sh @@ -16,6 +16,24 @@ GSTINSPECT="${GSTINSPECT:-gst-inspect-1.0}" GSTDISCOVER="${GSTDISCOVER:-gst-discoverer-1.0}" GSTLAUNCHFLAGS="${GSTLAUNCHFLAGS:--e -v -m}" +# Time bounds for anything that touches the hardware codecs. +# GST_PROBE_TIMEOUT bounds introspection (gst-inspect-1.0, gst-discoverer-1.0); +# GST_KILL_GRACE is how long a process gets to react to the first signal before +# it is SIGKILLed. +# +# The probe bound is deliberately generous: gst-inspect-1.0 legitimately takes +# tens of seconds when it builds a cold plugin registry, or when a codec driver +# retries a failing firmware load before giving up (~17s observed on +# x1e80100). It only has to be short enough that a run against wedged hardware +# still fits in the CI budget, which has_element's cache below takes care of. +GST_PROBE_TIMEOUT="${GST_PROBE_TIMEOUT:-60}" +GST_KILL_GRACE="${GST_KILL_GRACE:-10}" + +# Canonical gstreamer_run_bounded return codes, normalised across timeout(1) +# implementations (GNU coreutils and BusyBox report differently). +GST_RC_TIMEDOUT=124 # the command was stopped by the time bound +GST_RC_NO_TIMEOUT=125 # no usable timeout(1); the command was NOT run + # Optional env overrides (set by run.sh) # GST_ALSA_PLAYBACK_DEVICE=hw:0,0 # GST_ALSA_CAPTURE_DEVICE=hw:0,1 @@ -91,12 +109,152 @@ gstreamer_shared_recorded_dir() { gstreamer_shared_artifact_dir "AUDIO_SHARED_RECORDED_DIR" "audio-record-playback" "recorded" "$script_dir" "$outdir" } +# -------------------- Bounded command execution -------------------- +# gstreamer_signum +# Maps a signal name to its number. timeout(1) implementations differ in which +# signal names they accept, but every one of them accepts a number. +gstreamer_signum() { + case "$1" in + INT|SIGINT|2) printf '%s\n' 2 ;; + KILL|SIGKILL|9) printf '%s\n' 9 ;; + TERM|SIGTERM|15) printf '%s\n' 15 ;; + *) printf '%s\n' 15 ;; + esac +} + +# gstreamer_timeout_supported +# True when timeout(1) exists and understands the options this library needs. +# +# Only the short options -s and -k are used: GNU coreutils accepts both short +# and long forms, BusyBox (as shipped in most Yocto images) accepts only the +# short ones and fails outright on --signal=/--kill-after=. +# +# -k is required, not a nicety. Without the SIGKILL escalation a process that +# ignores the first signal is never stopped, which is exactly the hang this +# library exists to bound, so a timeout(1) lacking -k counts as unusable. +gstreamer_timeout_supported() { + if [ -z "${GST_TIMEOUT_OK:-}" ]; then + if command -v timeout >/dev/null 2>&1 && timeout -s 15 -k 1 1 true >/dev/null 2>&1; then + GST_TIMEOUT_OK=1 + else + GST_TIMEOUT_OK=0 + fi + fi + [ "$GST_TIMEOUT_OK" = "1" ] +} + +# gstreamer_run_bounded [args...] +# Runs under timeout(1), escalating to SIGKILL after GST_KILL_GRACE +# seconds if it does not react to . +# +# A process blocked in a V4L2 ioctl on a wedged hardware codec never reacts to +# the first signal. Without the escalation timeout(1) then waits for it forever, +# so the caller hangs until the CI harness times out the whole job instead of +# reporting a single failed test case. +# +# Returns the command's own exit status, GST_RC_TIMEDOUT if the bound stopped +# it, or GST_RC_NO_TIMEOUT if there is no usable timeout(1) - in which case the +# command is deliberately NOT run. Falling back to an unbounded run would +# reinstate the very hang this is here to prevent, so callers are expected to +# fail or skip instead. +# +# GST_BOUNDED_RAW_RC keeps the pre-normalisation status for diagnostics. +gstreamer_run_bounded() { + bounded_secs="$1" + bounded_sig="$2" + shift 2 + + if ! gstreamer_timeout_supported; then + log_error "No timeout(1) supporting -s/-k; refusing to run '$1' unbounded" + GST_BOUNDED_RAW_RC="$GST_RC_NO_TIMEOUT" + return "$GST_RC_NO_TIMEOUT" + fi + + bounded_signum=$(gstreamer_signum "$bounded_sig") + + timeout -s "$bounded_signum" -k "$GST_KILL_GRACE" "$bounded_secs" "$@" + bounded_rc=$? + GST_BOUNDED_RAW_RC="$bounded_rc" + + # Normalise "the bound stopped it" across implementations: + # GNU reports 124 when its signal ended the command, + # BusyBox reports 128+signal for the signal it delivered, + # both report 137 (128+SIGKILL) when the grace period had to escalate. + bounded_killed=$((128 + bounded_signum)) + if [ "$bounded_rc" = "124" ] || [ "$bounded_rc" = "137" ] || + [ "$bounded_rc" = "$bounded_killed" ]; then + bounded_rc="$GST_RC_TIMEDOUT" + fi + + return "$bounded_rc" +} + +# gstreamer_bounded_timed_out +# True when gstreamer_run_bounded had to stop the command. +gstreamer_bounded_timed_out() { + [ "$1" = "$GST_RC_TIMEDOUT" ] +} + +# gstreamer_bounded_no_timeout +# True when gstreamer_run_bounded refused to run the command because this +# system has no usable timeout(1). +gstreamer_bounded_no_timeout() { + [ "$1" = "$GST_RC_NO_TIMEOUT" ] +} + # -------------------- Element check -------------------- +# has_element +# True when the GStreamer element is registered and can be introspected. +# +# gst-inspect-1.0 opens every /dev/video* node while building or validating the +# plugin registry. If a hardware codec has wedged, that open blocks in the +# kernel and gst-inspect never returns. Bound it so a dead codec surfaces as +# "element not available" (SKIP/FAIL) rather than a hung test run. +# +# The answer is cached per element. Callers probe the same element repeatedly +# (run_encode_test asks once itself and once more via +# gstreamer_build_v4l2_encode_pipeline), and against wedged hardware every one +# of those probes would otherwise cost a full GST_PROBE_TIMEOUT. Caching keeps +# the worst case proportional to the number of distinct elements. +# +# Call gstreamer_reset_element_cache after anything that changes which elements +# exist, e.g. a video_ensure_stack hot switch that reloads the codec modules. has_element() { elem="$1" [ -n "$elem" ] || return 1 command -v "$GSTINSPECT" >/dev/null 2>&1 || return 1 - "$GSTINSPECT" "$elem" >/dev/null 2>&1 + + # Element names map to shell variable names, so reduce them to a safe charset. + has_element_key=$(printf '%s' "$elem" | tr -c 'A-Za-z0-9_' '_') + has_element_cached=$(eval "printf '%s' \"\${HAS_ELEMENT_CACHE_${has_element_key}:-}\"") + if [ -n "$has_element_cached" ]; then + return "$has_element_cached" + fi + + gstreamer_run_bounded "$GST_PROBE_TIMEOUT" TERM "$GSTINSPECT" "$elem" >/dev/null 2>&1 + has_element_rc=$? + + if gstreamer_bounded_no_timeout "$has_element_rc"; then + log_error "Cannot probe '$elem' without a usable timeout(1); treating it as unavailable" + has_element_rc=1 + elif gstreamer_bounded_timed_out "$has_element_rc"; then + log_warn "$GSTINSPECT timed out after ${GST_PROBE_TIMEOUT}s inspecting '$elem' (wedged codec device?)" + has_element_rc=1 + elif [ "$has_element_rc" -ne 0 ]; then + has_element_rc=1 + fi + + eval "HAS_ELEMENT_CACHE_${has_element_key}=\$has_element_rc" + return "$has_element_rc" +} + +# gstreamer_reset_element_cache +# Drops every cached has_element answer. Use after reloading codec modules or +# otherwise changing which GStreamer elements are registered. +gstreamer_reset_element_cache() { + for has_element_var in $(set | sed -n 's/^\(HAS_ELEMENT_CACHE_[A-Za-z0-9_]*\)=.*/\1/p'); do + unset "$has_element_var" + done } # -------------------- Pretty printing (multi-line) -------------------- @@ -390,7 +548,15 @@ gstreamer_log_clip_metadata() { : >"$metaLog" 2>/dev/null || true - "$GSTDISCOVER" "$clip" >"$metaLog" 2>&1 || true + gstreamer_run_bounded "$GST_PROBE_TIMEOUT" TERM "$GSTDISCOVER" "$clip" >"$metaLog" 2>&1 + clipmeta_rc=$? + if gstreamer_bounded_no_timeout "$clipmeta_rc"; then + log_warn "Skipping $GSTDISCOVER: no timeout(1) supporting -s/-k" + return 1 + fi + if gstreamer_bounded_timed_out "$clipmeta_rc"; then + log_warn "$GSTDISCOVER timed out after ${GST_PROBE_TIMEOUT}s on '$clip'" + fi log_info "Clip metadata ($GSTDISCOVER):" while IFS= read -r line; do @@ -522,25 +688,31 @@ gstreamer_run_gstlaunch_timeout() { secs="$1" pipe="$2" + # A non-numeric or non-positive bound is treated as invalid input and + # replaced with the default. There is deliberately no unbounded path here. case "$secs" in ''|*[!0-9]*) secs=10 ;; esac + [ "$secs" -gt 0 ] 2>/dev/null || secs=10 + command -v "$GSTBIN" >/dev/null 2>&1 || return 127 gstreamer_print_cmd_multiline "$pipe" - if [ "$secs" -gt 0 ] 2>/dev/null; then - if command -v timeout >/dev/null 2>&1; then - # shellcheck disable=SC2086 - # Send SIGINT instead of SIGTERM to trigger EOS via -e flag - timeout --signal=INT "$secs" "$GSTBIN" $GSTLAUNCHFLAGS $pipe - return $? + # shellcheck disable=SC2086 + # Send SIGINT instead of SIGTERM to trigger EOS via -e flag + gstreamer_run_bounded "$secs" INT "$GSTBIN" $GSTLAUNCHFLAGS $pipe + gstlaunch_rc=$? + + if gstreamer_bounded_no_timeout "$gstlaunch_rc"; then + log_error "Refusing to run $GSTBIN unbounded: no timeout(1) supporting -s/-k" + elif gstreamer_bounded_timed_out "$gstlaunch_rc"; then + if [ "$GST_BOUNDED_RAW_RC" = "137" ]; then + log_warn "$GSTBIN ignored SIGINT and was killed after ${secs}s + ${GST_KILL_GRACE}s (wedged codec device?)" else - log_warn "No timeout command available, running without timeout" + log_warn "$GSTBIN did not finish within ${secs}s" fi fi - # shellcheck disable=SC2086 - "$GSTBIN" $GSTLAUNCHFLAGS $pipe - return $? + return "$gstlaunch_rc" } # -------------------- Audio Record/Playback pipeline builders -------------------- From 95130b16341ae62800511323d798df59223bd609 Mon Sep 17 00:00:00 2001 From: Milosz Wasilewski Date: Wed, 26 Aug 2026 09:20:41 -0400 Subject: [PATCH 2/7] gstreamer: classify probe failures and reject unclean encodes has_element() collapsed every failure to 1, so a probe that had to be stopped was indistinguishable from an element that is simply absent. run_encode_test treats an empty encoder as SKIP, so a wedged codec was reported as "not applicable". Classify the outcome instead: GST_ELEM_OK, GST_ELEM_MISSING, GST_ELEM_TIMEOUT and GST_ELEM_NO_TIMEOUT. Every non-zero value still means "not usable", so boolean callers are unaffected, but the reason survives - including in the cache, which now stores the classification. gstreamer_v4l2_{encoder,decoder}_for_codec propagate it, and gstreamer_probe_unhealthy() tells callers which outcomes are hardware or environment failures. Video_Encode_Decode now reports FAIL for those and keeps SKIP for a genuinely missing element. run_encode_test logged the gst-launch status and then ignored it, deciding purely on log validation plus an output file larger than 1000 bytes. The encoded directory is shared between the test definitions in a job, so a stale artifact produced PASS. Delete the intended output before launching and fail immediately when the encode status is non-zero. log_warn and log_error write to stdout, and has_element runs inside command substitution, so the timeout warning was spliced into the element name and from there into the pipeline string - leaving $encoder non-empty and defeating both the old SKIP path and the new FAIL path. Send those two diagnostics to stderr. Signed-off-by: Milosz Wasilewski --- .../Video/Video_Encode_Decode/run.sh | 37 +++++- Runner/utils/lib_gstreamer.sh | 113 +++++++++++++----- 2 files changed, 121 insertions(+), 29 deletions(-) diff --git a/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh b/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh index a0e556ee..e2d15460 100755 --- a/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh +++ b/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh @@ -124,6 +124,10 @@ cleanup() { if ! pkill -P "$$" -x gst-launch-1.0 >/dev/null 2>&1; then pkill -x gst-launch-1.0 >/dev/null 2>&1 || true fi + # The element-probe cache is backed by files under TMPDIR; do not leave them. + if command -v gstreamer_reset_element_cache >/dev/null 2>&1; then + gstreamer_reset_element_cache + fi } trap cleanup INT TERM EXIT @@ -455,7 +459,15 @@ run_encode_test() { # Check if encoder is available encoder=$(gstreamer_v4l2_encoder_for_codec "$codec") + probe_rc=$? if [ -z "$encoder" ]; then + # A probe that timed out, or could not be bounded, means the codec is + # broken - not absent. Reporting SKIP there hides a hardware failure. + if gstreamer_probe_unhealthy "$probe_rc"; then + log_fail "$testname: FAIL (encoder probe failed for $codec, rc=$probe_rc)" + fail_count=$((fail_count + 1)) + return 1 + fi log_warn "Encoder not available for $codec" skip_count=$((skip_count + 1)) return 1 @@ -480,7 +492,12 @@ run_encode_test() { fi log_info "Pipeline: $pipeline" - + + # Never judge this run on a file left behind by an earlier one: the encoded + # directory is shared between the test definitions in a job, so a stale or + # partially written artifact would otherwise satisfy the size check below. + rm -f "$output_file" 2>/dev/null || true + # Run encoding if gstreamer_run_gstlaunch_timeout "$((duration + 10))" "$pipeline" >>"$test_log" 2>&1; then gstRc=0 @@ -489,7 +506,17 @@ run_encode_test() { fi log_info "Encode exit code: $gstRc" - + + # An encode that did not exit cleanly is a failure. videotestsrc delivers a + # fixed number of buffers and the pipeline EOSes well inside the bound, so a + # timeout or a signal here means the encoder never completed - unlike the + # playback cases, where running until the bound is the intended behaviour. + if [ "$gstRc" -ne 0 ]; then + log_fail "$testname: FAIL (rc=$gstRc)" + fail_count=$((fail_count + 1)) + return 1 + fi + # Check for GStreamer errors in log if ! gstreamer_validate_log "$test_log" "$testname"; then log_fail "$testname: FAIL (GStreamer errors detected)" @@ -530,7 +557,13 @@ run_decode_test() { # Check if decoder is available decoder=$(gstreamer_v4l2_decoder_for_codec "$codec") + probe_rc=$? if [ -z "$decoder" ]; then + if gstreamer_probe_unhealthy "$probe_rc"; then + log_fail "$testname: FAIL (decoder probe failed for $codec, rc=$probe_rc)" + fail_count=$((fail_count + 1)) + return 1 + fi log_warn "Decoder not available for $codec" skip_count=$((skip_count + 1)) return 1 diff --git a/Runner/utils/lib_gstreamer.sh b/Runner/utils/lib_gstreamer.sh index e67ac54c..693f9b77 100755 --- a/Runner/utils/lib_gstreamer.sh +++ b/Runner/utils/lib_gstreamer.sh @@ -34,6 +34,24 @@ GST_KILL_GRACE="${GST_KILL_GRACE:-10}" GST_RC_TIMEDOUT=124 # the command was stopped by the time bound GST_RC_NO_TIMEOUT=125 # no usable timeout(1); the command was NOT run +# Element-probe outcomes returned by has_element(). Every non-zero value still +# means "not usable", so boolean callers keep working, but the value says WHY: +# a genuinely absent element is a legitimate SKIP, whereas a probe that had to +# be stopped, or could not be bounded at all, points at broken hardware or a +# broken environment and must not be reported as "not applicable". +GST_ELEM_OK=0 +GST_ELEM_MISSING=1 +GST_ELEM_TIMEOUT=2 +GST_ELEM_NO_TIMEOUT=3 + +# Where has_element() remembers probe outcomes. This has to be a file, not a +# shell variable: the callers resolve elements inside command substitution +# (encoder=$(gstreamer_v4l2_encoder_for_codec ...)), which runs in a subshell, +# so any variable the probe set would be discarded on return and every +# resolution would pay the full GST_PROBE_TIMEOUT again. $$ is the invoking +# shell's PID and is stable across subshells, so one run shares one directory. +GST_ELEM_CACHE_DIR="${GST_ELEM_CACHE_DIR:-${TMPDIR:-/tmp}/gst-elem-cache-$$}" + # Optional env overrides (set by run.sh) # GST_ALSA_PLAYBACK_DEVICE=hw:0,0 # GST_ALSA_CAPTURE_DEVICE=hw:0,1 @@ -219,42 +237,63 @@ gstreamer_bounded_no_timeout() { # # Call gstreamer_reset_element_cache after anything that changes which elements # exist, e.g. a video_ensure_stack hot switch that reloads the codec modules. +# +# Returns GST_ELEM_OK, GST_ELEM_MISSING, GST_ELEM_TIMEOUT or GST_ELEM_NO_TIMEOUT; +# the classification is what gets cached, so a repeat probe keeps the reason. has_element() { elem="$1" [ -n "$elem" ] || return 1 command -v "$GSTINSPECT" >/dev/null 2>&1 || return 1 - # Element names map to shell variable names, so reduce them to a safe charset. + # Element names become file names, so reduce them to a safe charset. has_element_key=$(printf '%s' "$elem" | tr -c 'A-Za-z0-9_' '_') - has_element_cached=$(eval "printf '%s' \"\${HAS_ELEMENT_CACHE_${has_element_key}:-}\"") - if [ -n "$has_element_cached" ]; then - return "$has_element_cached" + has_element_cf="${GST_ELEM_CACHE_DIR}/${has_element_key}" + if [ -f "$has_element_cf" ]; then + has_element_cached=$(cat "$has_element_cf" 2>/dev/null) + case "$has_element_cached" in + ''|*[!0-9]*) : ;; # unreadable or corrupt: fall through and re-probe + *) return "$has_element_cached" ;; + esac fi gstreamer_run_bounded "$GST_PROBE_TIMEOUT" TERM "$GSTINSPECT" "$elem" >/dev/null 2>&1 has_element_rc=$? + # Diagnostics go to stderr: has_element runs inside command substitution + # (gstreamer_v4l2_encoder_for_codec and the pipeline builders capture its + # caller's stdout), so anything written to stdout here ends up spliced into + # the element name and, from there, into the pipeline string. if gstreamer_bounded_no_timeout "$has_element_rc"; then - log_error "Cannot probe '$elem' without a usable timeout(1); treating it as unavailable" - has_element_rc=1 + log_error "Cannot probe '$elem' without a usable timeout(1)" >&2 + has_element_rc="$GST_ELEM_NO_TIMEOUT" elif gstreamer_bounded_timed_out "$has_element_rc"; then - log_warn "$GSTINSPECT timed out after ${GST_PROBE_TIMEOUT}s inspecting '$elem' (wedged codec device?)" - has_element_rc=1 + log_warn "$GSTINSPECT timed out after ${GST_PROBE_TIMEOUT}s inspecting '$elem' (wedged codec device?)" >&2 + has_element_rc="$GST_ELEM_TIMEOUT" elif [ "$has_element_rc" -ne 0 ]; then - has_element_rc=1 + has_element_rc="$GST_ELEM_MISSING" fi - eval "HAS_ELEMENT_CACHE_${has_element_key}=\$has_element_rc" + # Best effort: if the cache cannot be written the probe still works, it just + # is not remembered. + mkdir -p "$GST_ELEM_CACHE_DIR" 2>/dev/null || true + printf '%s\n' "$has_element_rc" > "$has_element_cf" 2>/dev/null || true + return "$has_element_rc" } +# gstreamer_probe_unhealthy +# True when an element probe failed for a reason that indicates broken hardware +# or a broken environment rather than a genuinely absent element. Callers should +# report FAIL for these, not SKIP. +gstreamer_probe_unhealthy() { + [ "$1" = "$GST_ELEM_TIMEOUT" ] || [ "$1" = "$GST_ELEM_NO_TIMEOUT" ] +} + # gstreamer_reset_element_cache # Drops every cached has_element answer. Use after reloading codec modules or # otherwise changing which GStreamer elements are registered. gstreamer_reset_element_cache() { - for has_element_var in $(set | sed -n 's/^\(HAS_ELEMENT_CACHE_[A-Za-z0-9_]*\)=.*/\1/p'); do - unset "$has_element_var" - done + rm -rf "$GST_ELEM_CACHE_DIR" 2>/dev/null || true } # -------------------- Pretty printing (multi-line) -------------------- @@ -1050,25 +1089,33 @@ gstreamer_v4l2_encoder_for_codec() { codec="$1" case "$codec" in h264) - if has_element v4l2h264enc; then + has_element v4l2h264enc + probe_rc=$? + if [ "$probe_rc" -eq 0 ]; then printf '%s\n' "v4l2h264enc" - return 0 + return "$GST_ELEM_OK" fi + printf '%s\n' "" + return "$probe_rc" ;; h265|hevc) - if has_element v4l2h265enc; then + has_element v4l2h265enc + probe_rc=$? + if [ "$probe_rc" -eq 0 ]; then printf '%s\n' "v4l2h265enc" - return 0 + return "$GST_ELEM_OK" fi + printf '%s\n' "" + return "$probe_rc" ;; vp9) - # VP9 is decode-only, no encoder support + # VP9 is decode-only, no encoder support: genuinely absent, not broken. printf '%s\n' "" - return 1 + return "$GST_ELEM_MISSING" ;; esac printf '%s\n' "" - return 1 + return "$GST_ELEM_MISSING" } # gstreamer_v4l2_decoder_for_codec @@ -1078,26 +1125,38 @@ gstreamer_v4l2_decoder_for_codec() { codec="$1" case "$codec" in h264) - if has_element v4l2h264dec; then + has_element v4l2h264dec + probe_rc=$? + if [ "$probe_rc" -eq 0 ]; then printf '%s\n' "v4l2h264dec" - return 0 + return "$GST_ELEM_OK" fi + printf '%s\n' "" + return "$probe_rc" ;; h265|hevc) - if has_element v4l2h265dec; then + has_element v4l2h265dec + probe_rc=$? + if [ "$probe_rc" -eq 0 ]; then printf '%s\n' "v4l2h265dec" - return 0 + return "$GST_ELEM_OK" fi + printf '%s\n' "" + return "$probe_rc" ;; vp9) - if has_element v4l2vp9dec; then + has_element v4l2vp9dec + probe_rc=$? + if [ "$probe_rc" -eq 0 ]; then printf '%s\n' "v4l2vp9dec" - return 0 + return "$GST_ELEM_OK" fi + printf '%s\n' "" + return "$probe_rc" ;; esac printf '%s\n' "" - return 1 + return "$GST_ELEM_MISSING" } # gstreamer_container_ext_for_codec From ee4ceb5b4f9ca8c3ff8ed327e2c2ccb7ce641a18 Mon Sep 17 00:00:00 2001 From: Milosz Wasilewski Date: Wed, 26 Aug 2026 14:34:50 -0400 Subject: [PATCH 3/7] gstreamer: fail when a codec is absent because its driver did not come up The probe classification added earlier separates a wedged codec from an absent one, but it does not catch the case that motivated it. When the video firmware fails to load, the driver never registers its V4L2 element, so gst-inspect answers cleanly that the element does not exist. That is indistinguishable from a platform which genuinely has no such codec, and Video_Encode_Decode reports SKIP for both. LAVA jobs 382831 and 382832 show it plainly: two hamoa-iot-evk boards logged qcom-iris aa00000.video-codec: firmware download failed -16 qcom-iris aa00000.video-codec: core init failed and the suite still reported SKIP Only the kernel log separates "this platform has no such codec" from "the codec is there and broken", so consult it - but only for the V4L2 hardware codec lookups, and only when the element is absent. If the probe succeeded, or the element is absent on a machine with a quiet log, the result is unchanged. GST_ELEM_HW_FAULT joins the existing outcomes and gstreamer_probe_unhealthy treats it as a failure, so Video_Encode_Decode reports FAIL without any change to its own logic. The signature is deliberately narrow - hard firmware or init failures on a codec device - so that a transient session error the driver recovers from, or an unrelated GPU firmware message, does not turn a legitimate SKIP into a failure. gstreamer_probe_reason() turns the numeric outcome into a sentence, so the test log says why the codec was unusable rather than printing rc=4. Signed-off-by: Milosz Wasilewski --- .../Video/Video_Encode_Decode/run.sh | 4 +- Runner/utils/lib_gstreamer.sh | 53 ++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh b/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh index e2d15460..dcf803c3 100755 --- a/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh +++ b/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh @@ -464,7 +464,7 @@ run_encode_test() { # A probe that timed out, or could not be bounded, means the codec is # broken - not absent. Reporting SKIP there hides a hardware failure. if gstreamer_probe_unhealthy "$probe_rc"; then - log_fail "$testname: FAIL (encoder probe failed for $codec, rc=$probe_rc)" + log_fail "$testname: FAIL (encoder for $codec unusable: $(gstreamer_probe_reason "$probe_rc"))" fail_count=$((fail_count + 1)) return 1 fi @@ -560,7 +560,7 @@ run_decode_test() { probe_rc=$? if [ -z "$decoder" ]; then if gstreamer_probe_unhealthy "$probe_rc"; then - log_fail "$testname: FAIL (decoder probe failed for $codec, rc=$probe_rc)" + log_fail "$testname: FAIL (decoder for $codec unusable: $(gstreamer_probe_reason "$probe_rc"))" fail_count=$((fail_count + 1)) return 1 fi diff --git a/Runner/utils/lib_gstreamer.sh b/Runner/utils/lib_gstreamer.sh index 693f9b77..2e315b0c 100755 --- a/Runner/utils/lib_gstreamer.sh +++ b/Runner/utils/lib_gstreamer.sh @@ -43,6 +43,15 @@ GST_ELEM_OK=0 GST_ELEM_MISSING=1 GST_ELEM_TIMEOUT=2 GST_ELEM_NO_TIMEOUT=3 +GST_ELEM_HW_FAULT=4 + +# Kernel signatures of a video codec driver that failed to come up. A driver +# whose firmware will not load never registers its V4L2 element, so gst-inspect +# reports the element as simply absent - indistinguishable from a platform that +# genuinely has no such codec. Only the kernel log separates the two, and only +# the first is a failure. Kept deliberately narrow: hard init/firmware faults on +# a codec device, not transient session errors the driver recovers from. +GST_CODEC_FAULT_RE="${GST_CODEC_FAULT_RE:-(qcom-iris|qcom-venus|venus_core|video-codec).*(firmware download failed|core init failed|initializing firmware)}" # Where has_element() remembers probe outcomes. This has to be a file, not a # shell variable: the callers resolve elements inside command substitution @@ -286,7 +295,44 @@ has_element() { # or a broken environment rather than a genuinely absent element. Callers should # report FAIL for these, not SKIP. gstreamer_probe_unhealthy() { - [ "$1" = "$GST_ELEM_TIMEOUT" ] || [ "$1" = "$GST_ELEM_NO_TIMEOUT" ] + [ "$1" = "$GST_ELEM_TIMEOUT" ] || + [ "$1" = "$GST_ELEM_NO_TIMEOUT" ] || + [ "$1" = "$GST_ELEM_HW_FAULT" ] +} + +# gstreamer_probe_reason +# Human-readable explanation of a probe outcome, for test log messages. +gstreamer_probe_reason() { + case "$1" in + "$GST_ELEM_OK") printf '%s\n' "available" ;; + "$GST_ELEM_MISSING") printf '%s\n' "element not registered" ;; + "$GST_ELEM_TIMEOUT") printf '%s\n' "probe timed out, codec appears wedged" ;; + "$GST_ELEM_NO_TIMEOUT") printf '%s\n' "no usable timeout(1), probe refused" ;; + "$GST_ELEM_HW_FAULT") printf '%s\n' "codec driver reported a firmware or init failure" ;; + *) printf '%s\n' "unknown probe status $1" ;; + esac +} + +# gstreamer_codec_hw_faulted +# True when the kernel log shows a video codec driver failing to initialise. +gstreamer_codec_hw_faulted() { + command -v dmesg >/dev/null 2>&1 || return 1 + dmesg 2>/dev/null | grep -Eqi "$GST_CODEC_FAULT_RE" +} + +# gstreamer_refine_codec_probe +# Prints , upgraded from GST_ELEM_MISSING to GST_ELEM_HW_FAULT when the +# kernel log shows the codec driver failed to come up. Used only by the V4L2 +# hardware codec lookups: for those an absent element accompanied by a driver +# fault is a broken device, not an unsupported feature, and reporting SKIP +# there hides exactly the failure the test exists to catch. +gstreamer_refine_codec_probe() { + refine_rc="$1" + if [ "$refine_rc" = "$GST_ELEM_MISSING" ] && gstreamer_codec_hw_faulted; then + printf '%s\n' "$GST_ELEM_HW_FAULT" + return 0 + fi + printf '%s\n' "$refine_rc" } # gstreamer_reset_element_cache @@ -1095,6 +1141,7 @@ gstreamer_v4l2_encoder_for_codec() { printf '%s\n' "v4l2h264enc" return "$GST_ELEM_OK" fi + probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") printf '%s\n' "" return "$probe_rc" ;; @@ -1105,6 +1152,7 @@ gstreamer_v4l2_encoder_for_codec() { printf '%s\n' "v4l2h265enc" return "$GST_ELEM_OK" fi + probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") printf '%s\n' "" return "$probe_rc" ;; @@ -1131,6 +1179,7 @@ gstreamer_v4l2_decoder_for_codec() { printf '%s\n' "v4l2h264dec" return "$GST_ELEM_OK" fi + probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") printf '%s\n' "" return "$probe_rc" ;; @@ -1141,6 +1190,7 @@ gstreamer_v4l2_decoder_for_codec() { printf '%s\n' "v4l2h265dec" return "$GST_ELEM_OK" fi + probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") printf '%s\n' "" return "$probe_rc" ;; @@ -1151,6 +1201,7 @@ gstreamer_v4l2_decoder_for_codec() { printf '%s\n' "v4l2vp9dec" return "$GST_ELEM_OK" fi + probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") printf '%s\n' "" return "$probe_rc" ;; From a55e8a91f057958c0a8d0f593dc073d8ec85d606 Mon Sep 17 00:00:00 2001 From: Milosz Wasilewski Date: Wed, 26 Aug 2026 17:02:21 -0400 Subject: [PATCH 4/7] gstreamer: treat a codec driver that never binds as a fault gstreamer_codec_hw_faulted() only recognised a codec that had bound and then failed to bring up its firmware. A driver that never binds at all fails earlier than that and never prints any firmware wording: qcom-iris aa00000.video-codec: probe with driver qcom-iris failed with error -5 That is the only codec line such a board logs. The element is absent for the same reason as on a genuinely faulty board - the driver is not there to register it - but the old pattern found nothing, so the encode test reported SKIP ("Encoder not available for h264") and a dead video codec looked like a platform that simply has no encoder. Add the two probe-failure spellings the kernel uses. Verified against the kernel logs of 21 hamoa-iot-evk runs: the new alternatives match only the two boots that actually failed to probe, and add no match on any of the other 19, including healthy boots of the same board. Signed-off-by: Milosz Wasilewski --- Runner/utils/lib_gstreamer.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Runner/utils/lib_gstreamer.sh b/Runner/utils/lib_gstreamer.sh index 2e315b0c..e52fe8cf 100755 --- a/Runner/utils/lib_gstreamer.sh +++ b/Runner/utils/lib_gstreamer.sh @@ -46,12 +46,20 @@ GST_ELEM_NO_TIMEOUT=3 GST_ELEM_HW_FAULT=4 # Kernel signatures of a video codec driver that failed to come up. A driver -# whose firmware will not load never registers its V4L2 element, so gst-inspect -# reports the element as simply absent - indistinguishable from a platform that +# that does not come up never registers its V4L2 element, so gst-inspect reports +# the element as simply absent - indistinguishable from a platform that # genuinely has no such codec. Only the kernel log separates the two, and only -# the first is a failure. Kept deliberately narrow: hard init/firmware faults on -# a codec device, not transient session errors the driver recovers from. -GST_CODEC_FAULT_RE="${GST_CODEC_FAULT_RE:-(qcom-iris|qcom-venus|venus_core|video-codec).*(firmware download failed|core init failed|initializing firmware)}" +# the first is a failure. Kept deliberately narrow: hard faults that leave the +# codec unusable, not transient session errors the driver recovers from. +# +# Two distinct shapes have to be covered, because they fail at different stages: +# - the driver binds, then cannot load or start its firmware +# ("firmware download failed", "core init failed", "initializing firmware") +# - the driver never binds at all ("probe with driver qcom-iris failed with +# error -5"), so it never reaches the point of mentioning firmware +# The second shape prints no firmware wording whatsoever, so matching only the +# first silently reports a dead codec as an unsupported one. +GST_CODEC_FAULT_RE="${GST_CODEC_FAULT_RE:-(qcom-iris|qcom-venus|venus_core|video-codec).*(firmware download failed|core init failed|initializing firmware|probe with driver [^ ]+ failed|probe of [^ ]+ failed)}" # Where has_element() remembers probe outcomes. This has to be a file, not a # shell variable: the callers resolve elements inside command substitution @@ -308,13 +316,14 @@ gstreamer_probe_reason() { "$GST_ELEM_MISSING") printf '%s\n' "element not registered" ;; "$GST_ELEM_TIMEOUT") printf '%s\n' "probe timed out, codec appears wedged" ;; "$GST_ELEM_NO_TIMEOUT") printf '%s\n' "no usable timeout(1), probe refused" ;; - "$GST_ELEM_HW_FAULT") printf '%s\n' "codec driver reported a firmware or init failure" ;; + "$GST_ELEM_HW_FAULT") printf '%s\n' "codec driver failed to come up (firmware, core init or probe)" ;; *) printf '%s\n' "unknown probe status $1" ;; esac } # gstreamer_codec_hw_faulted -# True when the kernel log shows a video codec driver failing to initialise. +# True when the kernel log shows a video codec driver failing to come up, +# either by failing to load its firmware or by failing to probe at all. gstreamer_codec_hw_faulted() { command -v dmesg >/dev/null 2>&1 || return 1 dmesg 2>/dev/null | grep -Eqi "$GST_CODEC_FAULT_RE" From 41cc2ddd2e6d1a6ed0236618c7048481cd575b7a Mon Sep 17 00:00:00 2001 From: Milosz Wasilewski Date: Fri, 28 Aug 2026 12:04:44 -0400 Subject: [PATCH 5/7] gstreamer: confine the probe cache to this process's own directory gstreamer_reset_element_cache() handed GST_ELEM_CACHE_DIR to rm -rf, and the variable was read from the environment. The reset runs from the suite's EXIT trap, so a stray value turned routine cleanup into an unbounded recursive delete of whatever that path named. Derive the path internally and stop consulting the environment for it, so nothing outside this library chooses what the reset deletes; a caller that needs a different location sets TMPDIR. The reset additionally re-derives the owned path and refuses anything else, since the variable can still be reassigned after the library is sourced, and removes entries individually by the name shape has_element() writes rather than recursing, so an unrelated file in a shared TMPDIR is never a candidate. rmdir() then drops the directory only when nothing else is in it. Signed-off-by: Milosz Wasilewski --- Runner/utils/lib_gstreamer.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/Runner/utils/lib_gstreamer.sh b/Runner/utils/lib_gstreamer.sh index e52fe8cf..60679b6f 100755 --- a/Runner/utils/lib_gstreamer.sh +++ b/Runner/utils/lib_gstreamer.sh @@ -67,7 +67,11 @@ GST_CODEC_FAULT_RE="${GST_CODEC_FAULT_RE:-(qcom-iris|qcom-venus|venus_core|video # so any variable the probe set would be discarded on return and every # resolution would pay the full GST_PROBE_TIMEOUT again. $$ is the invoking # shell's PID and is stable across subshells, so one run shares one directory. -GST_ELEM_CACHE_DIR="${GST_ELEM_CACHE_DIR:-${TMPDIR:-/tmp}/gst-elem-cache-$$}" +# Deliberately not read from the environment. gstreamer_reset_element_cache() +# deletes from this directory and runs from the suite's EXIT trap, so the path +# it clears has to be one this process derived, not one a caller can point +# elsewhere. Callers that want a different location get a different TMPDIR. +GST_ELEM_CACHE_DIR="${TMPDIR:-/tmp}/gst-elem-cache-$$" # Optional env overrides (set by run.sh) # GST_ALSA_PLAYBACK_DEVICE=hw:0,0 @@ -347,8 +351,28 @@ gstreamer_refine_codec_probe() { # gstreamer_reset_element_cache # Drops every cached has_element answer. Use after reloading codec modules or # otherwise changing which GStreamer elements are registered. +# Clears the probe answers this process cached. Refuses any path other than the +# one derived at load time: the variable is a plain shell variable that a caller +# could reassign, and this runs from the suite's EXIT trap, so a stray value +# must not turn cleanup into a delete of whatever that path names. Entries are +# matched against the shape has_element() writes, so nothing else in a shared +# TMPDIR is a candidate, and rmdir() removes the directory only if it is empty. gstreamer_reset_element_cache() { - rm -rf "$GST_ELEM_CACHE_DIR" 2>/dev/null || true + reset_owned="${TMPDIR:-/tmp}/gst-elem-cache-$$" + if [ "$GST_ELEM_CACHE_DIR" != "$reset_owned" ]; then + log_warn "Refusing to clear '$GST_ELEM_CACHE_DIR': not this process's cache" >&2 + return 1 + fi + [ -d "$GST_ELEM_CACHE_DIR" ] || return 0 + for reset_f in "$GST_ELEM_CACHE_DIR"/*; do + [ -f "$reset_f" ] || continue + case "${reset_f##*/}" in + *[!A-Za-z0-9_]*) continue ;; + esac + rm -f "$reset_f" 2>/dev/null || true + done + rmdir "$GST_ELEM_CACHE_DIR" 2>/dev/null || true + return 0 } # -------------------- Pretty printing (multi-line) -------------------- From 731e55db9c8bf248dde0ccf9eab55c0568604dee Mon Sep 17 00:00:00 2001 From: Milosz Wasilewski Date: Fri, 28 Aug 2026 12:05:34 -0400 Subject: [PATCH 6/7] gstreamer: classify codec faults from one authoritative dmesg snapshot gstreamer_codec_hw_faulted() read the live ring buffer on every call. It bypassed scan_dmesg_errors(), left nothing on disk showing why a codec had been classified as faulted, and re-read the log once per probed element. Take the snapshot once per run through scan_dmesg_errors() and classify against that file. The capture is the run's single authoritative one: the codec verdicts and the suite's end-of-run dmesg check now report from the same evidence instead of scanning the buffer twice with two patterns, so what the final check prints is what the verdicts were based on. The suite widens GST_CODEC_MODULE_RE to the modules it cares about and points GST_CODEC_DMESG_DIR at its own dmesg directory, so dmesg_snapshot.log and dmesg_errors.log land with the run's other artifacts. The snapshot has no cleanup hook on purpose. It is evidence, and the EXIT trap that clears the element cache must not take it with it; element-cache invalidation and the dmesg lifecycle are deliberately separate. The module pattern has to reach up to the colon. The driver name and the device node share one field ("qcom-iris aa00000.video-codec:"), and scan_dmesg_errors() anchors on ^[ts] (module):, so a bare alternation never matches. Checked against the real helper: all three fault shapes are captured and the benign dummy-regulator line stays excluded. scan_dmesg_errors() logs on stdout and the codec lookups run inside command substitution, so its output goes to stderr here. On stdout it would be spliced into the element name the caller is resolving. Signed-off-by: Milosz Wasilewski --- .../Video/Video_Encode_Decode/run.sh | 22 ++++--- Runner/utils/lib_gstreamer.sh | 57 ++++++++++++++++++- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh b/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh index dcf803c3..c47c9f46 100755 --- a/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh +++ b/Runner/suites/Multimedia/GSTreamer/Video/Video_Encode_Decode/run.sh @@ -67,6 +67,13 @@ if ! mkdir -p "$OUTDIR" "$DMESG_DIR" "$ENCODED_DIR"; then exit 0 fi +# Keep the codec-fault evidence with the run's artifacts, and widen the module +# filter to the set this suite cares about so the single capture below serves +# both the codec classification and the end-of-run check. +GST_CODEC_DMESG_DIR="$DMESG_DIR" +GST_CODEC_MODULE_RE="qcom-iris[^:]*|qcom-venus[^:]*|venus_core[^:]*|[^ ]*video-codec|venus[^:]*|vcodec[^:]*|v4l2[^:]*|video[^:]*|gstreamer[^:]*" +export GST_CODEC_DMESG_DIR GST_CODEC_MODULE_RE + : >"$RES_FILE" : >"$GST_LOG" @@ -741,20 +748,19 @@ log_info "==========================================" log_info "DMESG ERROR SCAN" log_info "==========================================" -# Scan for video-related errors in dmesg -module_regex="venus|vcodec|v4l2|video|gstreamer" -exclude_regex="dummy regulator|supply [^ ]+ not found|using dummy regulator" - -if command -v scan_dmesg_errors >/dev/null 2>&1; then - scan_dmesg_errors "$DMESG_DIR" "$module_regex" "$exclude_regex" || true - +# Report from the run's single dmesg capture rather than re-reading the live +# buffer. gstreamer_codec_dmesg_snapshot() takes it the first time a codec is +# classified and is a no-op afterwards, so the errors reported here are the same +# evidence the codec verdicts were based on. +if gstreamer_codec_dmesg_snapshot >/dev/null 2>&1; then + log_info "dmesg snapshot: $DMESG_DIR/dmesg_snapshot.log" if [ -s "$DMESG_DIR/dmesg_errors.log" ]; then log_warn "dmesg scan found video-related warnings or errors in $DMESG_DIR/dmesg_errors.log" else log_info "No relevant video-related errors found in dmesg" fi else - log_info "scan_dmesg_errors not available, skipping dmesg scan" + log_info "dmesg snapshot unavailable, skipping dmesg scan" fi # -------------------- Summary -------------------- diff --git a/Runner/utils/lib_gstreamer.sh b/Runner/utils/lib_gstreamer.sh index 60679b6f..cdfd870b 100755 --- a/Runner/utils/lib_gstreamer.sh +++ b/Runner/utils/lib_gstreamer.sh @@ -73,6 +73,22 @@ GST_CODEC_FAULT_RE="${GST_CODEC_FAULT_RE:-(qcom-iris|qcom-venus|venus_core|video # elsewhere. Callers that want a different location get a different TMPDIR. GST_ELEM_CACHE_DIR="${TMPDIR:-/tmp}/gst-elem-cache-$$" +# Where the codec-fault evidence is kept, and what scan_dmesg_errors() is asked +# to look at when producing it. The module pattern has to tolerate the driver +# name and the device node sharing one field ("qcom-iris aa00000.video-codec:"), +# which is why each alternative reaches up to the colon. +# +# Declared after GST_ELEM_CACHE_DIR because the default derives from it: read +# any earlier it would expand to the empty string, putting the snapshot at +# /dmesg_snapshot.log and quietly disabling fault classification. +# +# Defaults to the probe cache dir so the library is usable on its own; the suite +# points GST_CODEC_DMESG_DIR at its own dmesg directory so the snapshot and the +# filtered error log are kept with the rest of the run's artifacts. +GST_CODEC_DMESG_DIR="${GST_CODEC_DMESG_DIR:-$GST_ELEM_CACHE_DIR}" +GST_CODEC_MODULE_RE="${GST_CODEC_MODULE_RE:-qcom-iris[^:]*|qcom-venus[^:]*|venus_core[^:]*|[^ ]*video-codec}" +GST_CODEC_DMESG_EXCLUDE="${GST_CODEC_DMESG_EXCLUDE:-dummy regulator|supply [^ ]+ not found|using dummy regulator}" + # Optional env overrides (set by run.sh) # GST_ALSA_PLAYBACK_DEVICE=hw:0,0 # GST_ALSA_CAPTURE_DEVICE=hw:0,1 @@ -325,12 +341,49 @@ gstreamer_probe_reason() { esac } +# gstreamer_codec_dmesg_snapshot +# Prints the path of the run's dmesg snapshot, capturing it once and reusing it +# thereafter. This is the single authoritative capture for the run: the codec +# probes classify against it and the suite's end-of-run check reports from the +# error log taken alongside it, so the live ring buffer is read exactly once no +# matter how many elements are probed. +# +# Goes through scan_dmesg_errors() so the snapshot and the filtered error log +# are the suite's standard artifacts rather than a private copy, which keeps the +# evidence for a classification on disk next to the run's other logs. Falls back +# to a plain capture only where that helper is unavailable. +# +# scan_dmesg_errors() logs on stdout, and the codec lookups run inside command +# substitution, so its output is sent to stderr here; letting it reach stdout +# would splice log text into the element name the caller is resolving. +gstreamer_codec_dmesg_snapshot() { + [ -n "$GST_CODEC_DMESG_DIR" ] || return 1 + snap_file="$GST_CODEC_DMESG_DIR/dmesg_snapshot.log" + if [ -s "$snap_file" ]; then + printf '%s\n' "$snap_file" + return 0 + fi + command -v dmesg >/dev/null 2>&1 || return 1 + mkdir -p "$GST_CODEC_DMESG_DIR" 2>/dev/null || return 1 + if command -v scan_dmesg_errors >/dev/null 2>&1; then + scan_dmesg_errors "$GST_CODEC_DMESG_DIR" "$GST_CODEC_MODULE_RE" \ + "$GST_CODEC_DMESG_EXCLUDE" >&2 2>&2 || true + fi + if [ ! -s "$snap_file" ]; then + dmesg > "$snap_file" 2>/dev/null || true + fi + [ -s "$snap_file" ] || return 1 + printf '%s\n' "$snap_file" +} + # gstreamer_codec_hw_faulted # True when the kernel log shows a video codec driver failing to come up, # either by failing to load its firmware or by failing to probe at all. +# Reads the run's snapshot, so every probe classifies against the same evidence +# and that evidence outlives the run. gstreamer_codec_hw_faulted() { - command -v dmesg >/dev/null 2>&1 || return 1 - dmesg 2>/dev/null | grep -Eqi "$GST_CODEC_FAULT_RE" + hwf_snap=$(gstreamer_codec_dmesg_snapshot) || return 1 + grep -Eqi "$GST_CODEC_FAULT_RE" "$hwf_snap" } # gstreamer_refine_codec_probe From 11a2069209bb5037600c12e340164103ff2ea448 Mon Sep 17 00:00:00 2001 From: Milosz Wasilewski Date: Fri, 28 Aug 2026 12:07:23 -0400 Subject: [PATCH 7/7] gstreamer: stop re-probing once a codec probe proves unusable Bounding each probe individually is not enough. A wedged codec device does not wedge one element: gst-inspect hangs the same way for every element the driver would have registered, so a run that resolves an encoder and two decoders pays GST_PROBE_TIMEOUT three times over. The per-element cache does not help, because each lookup asks about a different element. That is how the outer LAVA timeout still gets consumed even though no single probe runs unbounded. Record the first outcome that means the probe could not complete - timed out, or no usable timeout(1) - as a codec-wide verdict, and answer later hardware codec lookups from it without starting another gst-inspect. Only those two outcomes are recorded: a missing element is specific to that element and says nothing about whether the next one can be probed. Measured against a signal-ignoring gst-inspect with a 3s bound, three lookups: 12s and three probes before, 4s and one probe after. The five V4L2 lookup arms were the same block of code five times over and now share one helper, which is what makes the verdict apply to all of them rather than to whichever arm was edited. Signed-off-by: Milosz Wasilewski --- Runner/utils/lib_gstreamer.sh | 116 +++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/Runner/utils/lib_gstreamer.sh b/Runner/utils/lib_gstreamer.sh index cdfd870b..ffdd090b 100755 --- a/Runner/utils/lib_gstreamer.sh +++ b/Runner/utils/lib_gstreamer.sh @@ -401,6 +401,67 @@ gstreamer_refine_codec_probe() { printf '%s\n' "$refine_rc" } +# The codec-wide verdict, kept next to the per-element answers. A wedged codec +# device does not wedge one element: gst-inspect hangs the same way for every +# element the driver would have registered, so probing each one costs a further +# GST_PROBE_TIMEOUT. A run resolving an encoder and two decoders pays it three +# times over, which is how the outer LAVA timeout gets consumed even though each +# probe is individually bounded. Recording the first unusable outcome lets the +# rest of the lookups answer immediately. +# +# Named so it matches the entry shape the cache reset deletes, and so it can +# never collide with a sanitised GStreamer element name. +GST_CODEC_STATE_KEY="_codec_state" + +# gstreamer_codec_state +# Prints the recorded codec-wide probe status, or fails if none is recorded. +gstreamer_codec_state() { + state_f="$GST_ELEM_CACHE_DIR/$GST_CODEC_STATE_KEY" + [ -f "$state_f" ] || return 1 + state_v=$(cat "$state_f" 2>/dev/null) + case "$state_v" in + ''|*[!0-9]*) return 1 ;; + esac + printf '%s\n' "$state_v" +} + +# gstreamer_mark_codec_state +# Records a codec-wide probe status. Best effort: if it cannot be written the +# probes still work, they are just not short-circuited. +gstreamer_mark_codec_state() { + mkdir -p "$GST_ELEM_CACHE_DIR" 2>/dev/null || true + printf '%s\n' "$1" > "$GST_ELEM_CACHE_DIR/$GST_CODEC_STATE_KEY" 2>/dev/null || true +} + +# gstreamer_probe_codec_element +# Resolves one V4L2 hardware codec element. Prints the element name when it is +# usable and nothing otherwise, returning the GST_ELEM_* classification. +# +# Answers from the codec-wide verdict when one is already recorded, so the +# second and later lookups in a run with a wedged codec cost nothing. Only +# outcomes that mean the probe could not complete are recorded that way: a +# missing element is specific to that element and says nothing about the rest. +gstreamer_probe_codec_element() { + probe_elem="$1" + if probe_rc=$(gstreamer_codec_state); then + printf '%s\n' "" + return "$probe_rc" + fi + has_element "$probe_elem" + probe_rc=$? + if [ "$probe_rc" -eq 0 ]; then + printf '%s\n' "$probe_elem" + return "$GST_ELEM_OK" + fi + if [ "$probe_rc" = "$GST_ELEM_TIMEOUT" ] || [ "$probe_rc" = "$GST_ELEM_NO_TIMEOUT" ]; then + log_warn "Codec probe unusable ($(gstreamer_probe_reason "$probe_rc")); skipping further hardware codec probes this run" >&2 + gstreamer_mark_codec_state "$probe_rc" + fi + probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") + printf '%s\n' "" + return "$probe_rc" +} + # gstreamer_reset_element_cache # Drops every cached has_element answer. Use after reloading codec modules or # otherwise changing which GStreamer elements are registered. @@ -1221,26 +1282,12 @@ gstreamer_v4l2_encoder_for_codec() { codec="$1" case "$codec" in h264) - has_element v4l2h264enc - probe_rc=$? - if [ "$probe_rc" -eq 0 ]; then - printf '%s\n' "v4l2h264enc" - return "$GST_ELEM_OK" - fi - probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") - printf '%s\n' "" - return "$probe_rc" + gstreamer_probe_codec_element v4l2h264enc + return $? ;; h265|hevc) - has_element v4l2h265enc - probe_rc=$? - if [ "$probe_rc" -eq 0 ]; then - printf '%s\n' "v4l2h265enc" - return "$GST_ELEM_OK" - fi - probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") - printf '%s\n' "" - return "$probe_rc" + gstreamer_probe_codec_element v4l2h265enc + return $? ;; vp9) # VP9 is decode-only, no encoder support: genuinely absent, not broken. @@ -1259,37 +1306,16 @@ gstreamer_v4l2_decoder_for_codec() { codec="$1" case "$codec" in h264) - has_element v4l2h264dec - probe_rc=$? - if [ "$probe_rc" -eq 0 ]; then - printf '%s\n' "v4l2h264dec" - return "$GST_ELEM_OK" - fi - probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") - printf '%s\n' "" - return "$probe_rc" + gstreamer_probe_codec_element v4l2h264dec + return $? ;; h265|hevc) - has_element v4l2h265dec - probe_rc=$? - if [ "$probe_rc" -eq 0 ]; then - printf '%s\n' "v4l2h265dec" - return "$GST_ELEM_OK" - fi - probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") - printf '%s\n' "" - return "$probe_rc" + gstreamer_probe_codec_element v4l2h265dec + return $? ;; vp9) - has_element v4l2vp9dec - probe_rc=$? - if [ "$probe_rc" -eq 0 ]; then - printf '%s\n' "v4l2vp9dec" - return "$GST_ELEM_OK" - fi - probe_rc=$(gstreamer_refine_codec_probe "$probe_rc") - printf '%s\n' "" - return "$probe_rc" + gstreamer_probe_codec_element v4l2vp9dec + return $? ;; esac printf '%s\n' ""