From 569c7cb5a74733bf506bf7fa0db8db4b32d298ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E7=BA=A7?= =?UTF-8?q?=E5=96=9C=E6=AC=A2=E4=BD=A0=E7=9A=84=E8=BE=BE=E5=A6=AE=E5=A8=85?= <176143450+My-Denia@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:00:43 +0800 Subject: [PATCH 1/4] fix: snap illegal AAC sample rates to 48 kHz on Windows --- electron/native/wgc-capture/CMakeLists.txt | 22 ++ .../wgc-capture/src/audio_sample_utils.cpp | 23 +- .../src/audio_sample_utils_test.cpp | 325 ++++++++++++++++++ electron/native/wgc-capture/src/main.cpp | 28 +- .../native/wgc-capture/src/mf_encoder.cpp | 75 +++- electron/native/wgc-capture/src/mf_encoder.h | 13 +- scripts/build-windows-wgc-helper.mjs | 7 + 7 files changed, 485 insertions(+), 8 deletions(-) create mode 100644 electron/native/wgc-capture/src/audio_sample_utils_test.cpp diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index c2947df77..99041a02b 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -89,3 +89,25 @@ target_link_libraries(cursor-sampler PRIVATE gdi32 gdiplus ) + +add_executable(audio_sample_utils_test + src/audio_sample_utils.cpp + src/audio_sample_utils.h + src/audio_sample_utils_test.cpp +) + +target_compile_definitions(audio_sample_utils_test PRIVATE + NOMINMAX + WIN32_LEAN_AND_MEAN + _WIN32_WINNT=0x0A00 +) + +target_compile_options(audio_sample_utils_test PRIVATE /EHsc /W4 /utf-8) + +target_link_libraries(audio_sample_utils_test PRIVATE + mf + mfplat + mfreadwrite + mfuuid + ole32 +) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp index 5e60860c9..26fb57df5 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -87,6 +87,21 @@ double readMappedChannel(const BYTE* source, const AudioInputFormat& format, siz return readSampleAsDouble(source, format, frameIndex, std::min(targetChannel, format.channels - 1)); } +UINT32 aacCompatibleSampleRate(UINT32 sampleRate) { + constexpr UINT32 kAacSampleRates[] = { + 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000, + }; + if (sampleRate == 0) { + return 48000; + } + for (UINT32 rate : kAacSampleRates) { + if (sampleRate == rate) { + return rate; + } + } + return 48000; +} + } // namespace constexpr int64_t HnsPerSecond = 10'000'000; @@ -100,10 +115,16 @@ bool sameAudioFormatForMixing(const AudioInputFormat& left, const AudioInputForm left.avgBytesPerSec == right.avgBytesPerSec; } +// Microsoft AAC encoder (MFAudioFormat_AAC) sample rates. WASAPI loopback +// often reports 96000 or 192000; those are legal PCM mix rates but not AAC +// input rates, and SetInputMediaType then fails with MF_E_INVALIDMEDIATYPE +// (0xc00d36b4). Keep legal rates as-is so a working 44100/48000 path is +// unchanged; snap everything else (including 0) to 48000. The mixer already +// resamples through convertAudioWithGain when the source rate differs. AudioInputFormat makeAacCompatibleAudioFormat(const AudioInputFormat& source) { AudioInputFormat format{}; format.subtype = MFAudioFormat_PCM; - format.sampleRate = source.sampleRate > 0 ? source.sampleRate : 48000; + format.sampleRate = aacCompatibleSampleRate(source.sampleRate); format.channels = 2; format.bitsPerSample = 16; format.blockAlign = format.channels * (format.bitsPerSample / 8); diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp new file mode 100644 index 000000000..81f5cfb78 --- /dev/null +++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp @@ -0,0 +1,325 @@ +#include "audio_sample_utils.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +int g_ran = 0; +int g_failed = 0; + +AudioInputFormat makeFormat( + GUID subtype, + UINT32 sampleRate, + UINT32 channels, + UINT32 bitsPerSample) { + AudioInputFormat format{}; + format.subtype = subtype; + format.sampleRate = sampleRate; + format.channels = channels; + format.bitsPerSample = bitsPerSample; + format.blockAlign = channels * (bitsPerSample / 8); + format.avgBytesPerSec = sampleRate * format.blockAlign; + return format; +} + +void expect(const char* name, bool ok, const std::string& detail) { + g_ran += 1; + if (ok) { + std::cout << "PASS " << name << "\n"; + return; + } + g_failed += 1; + std::cout << "FAIL " << name << " " << detail << "\n"; +} + +std::string describe(const AudioInputFormat& format) { + return "sampleRate=" + std::to_string(format.sampleRate) + + " channels=" + std::to_string(format.channels) + + " bits=" + std::to_string(format.bitsPerSample); +} + +std::wstring tempMp4Path() { + wchar_t dir[MAX_PATH]{}; + GetTempPathW(MAX_PATH, dir); + return std::wstring(dir) + L"openscreen-mf-aac-probe-" + + std::to_wstring(GetCurrentProcessId()) + L".mp4"; +} + +HRESULT trySetAacPcmRate(UINT32 sampleRate, IMFAttributes* attributes = nullptr) { + const std::wstring path = tempMp4Path(); + DeleteFileW(path.c_str()); + + Microsoft::WRL::ComPtr writer; + HRESULT hr = MFCreateSinkWriterFromURL(path.c_str(), nullptr, attributes, &writer); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr outputType; + hr = MFCreateMediaType(&outputType); + if (FAILED(hr)) { + return hr; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + outputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + outputType->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + outputType->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + outputType->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + outputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, 24000); + outputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + DWORD streamIndex = 0; + hr = writer->AddStream(outputType.Get(), &streamIndex); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr inputType; + hr = MFCreateMediaType(&inputType); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + inputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); + inputType->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + inputType->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + inputType->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + inputType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, 4); + inputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, sampleRate * 4); + inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + + hr = writer->SetInputMediaType(streamIndex, inputType.Get(), nullptr); + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; +} + +// Same rates as trySetAacPcmRate, but with an H.264 stream first — the helper's +// topology. Distinguishes "96 kHz AAC is illegal" from "audio-only MP4 sink +// writer refuses this type". +HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { + const std::wstring path = tempMp4Path(); + DeleteFileW(path.c_str()); + + Microsoft::WRL::ComPtr writer; + HRESULT hr = MFCreateSinkWriterFromURL(path.c_str(), nullptr, nullptr, &writer); + if (FAILED(hr)) { + return hr; + } + + Microsoft::WRL::ComPtr videoOut; + hr = MFCreateMediaType(&videoOut); + if (FAILED(hr)) { + return hr; + } + videoOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + videoOut->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + videoOut->SetUINT32(MF_MT_AVG_BITRATE, 1'000'000); + videoOut->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + MFSetAttributeSize(videoOut.Get(), MF_MT_FRAME_SIZE, 320, 240); + MFSetAttributeRatio(videoOut.Get(), MF_MT_FRAME_RATE, 30, 1); + MFSetAttributeRatio(videoOut.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + + DWORD videoIndex = 0; + hr = writer->AddStream(videoOut.Get(), &videoIndex); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr audioOut; + hr = MFCreateMediaType(&audioOut); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + audioOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + audioOut->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + audioOut->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + audioOut->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + audioOut->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + audioOut->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, 24000); + audioOut->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + DWORD audioIndex = 0; + hr = writer->AddStream(audioOut.Get(), &audioIndex); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr videoIn; + hr = MFCreateMediaType(&videoIn); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + videoIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + videoIn->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + videoIn->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + videoIn->SetUINT32(MF_MT_DEFAULT_STRIDE, 320 * 4); + MFSetAttributeSize(videoIn.Get(), MF_MT_FRAME_SIZE, 320, 240); + MFSetAttributeRatio(videoIn.Get(), MF_MT_FRAME_RATE, 30, 1); + MFSetAttributeRatio(videoIn.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); + hr = writer->SetInputMediaType(videoIndex, videoIn.Get(), nullptr); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr audioIn; + hr = MFCreateMediaType(&audioIn); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + audioIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + audioIn->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); + audioIn->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, 2); + audioIn->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, sampleRate); + audioIn->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, 16); + audioIn->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, 4); + audioIn->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, sampleRate * 4); + audioIn->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + hr = writer->SetInputMediaType(audioIndex, audioIn.Get(), nullptr); + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; +} + +} // namespace + +int main() { + const AudioInputFormat diagnostic = makeFormat(MFAudioFormat_Float, 96000, 8, 32); + const AudioInputFormat snapped = makeAacCompatibleAudioFormat(diagnostic); + expect( + "diag-96000-8ch", + snapped.sampleRate == 48000 && snapped.channels == 2 && snapped.bitsPerSample == 16 && + snapped.subtype == MFAudioFormat_PCM, + describe(snapped)); + + const AudioInputFormat keep48000 = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 48000, 2, 16)); + expect("keep-48000", keep48000.sampleRate == 48000, describe(keep48000)); + + const AudioInputFormat keep44100 = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 44100, 2, 16)); + expect("keep-44100", keep44100.sampleRate == 44100, describe(keep44100)); + + const AudioInputFormat zeroRate = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 0, 2, 16)); + expect("zero-rate", zeroRate.sampleRate == 48000, describe(zeroRate)); + + const AudioInputFormat keep32000 = + makeAacCompatibleAudioFormat(makeFormat(MFAudioFormat_PCM, 32000, 2, 16)); + expect("keep-32000", keep32000.sampleRate == 32000, describe(keep32000)); + + const AudioInputFormat source96k = makeFormat(MFAudioFormat_PCM, 96000, 2, 16); + const AudioInputFormat target48k = makeAacCompatibleAudioFormat(source96k); + const UINT32 sourceFrames = 96000; + std::vector source(static_cast(sourceFrames) * source96k.blockAlign, 0); + auto* samples = reinterpret_cast(source.data()); + for (UINT32 frame = 0; frame < sourceFrames; frame += 1) { + samples[frame * 2] = static_cast(frame % 32767); + samples[frame * 2 + 1] = static_cast((frame * 3) % 32767); + } + std::vector converted; + convertAudioWithGain( + source.data(), + static_cast(source.size()), + source96k, + target48k, + 1.0, + converted); + const size_t convertedFrames = + target48k.blockAlign == 0 ? 0 : converted.size() / target48k.blockAlign; + const bool frameCountOk = + convertedFrames == 48000 || convertedFrames == 47999 || convertedFrames == 48001; + expect( + "resample-frame-count", + target48k.sampleRate == 48000 && frameCountOk, + "frames=" + std::to_string(convertedFrames) + " " + describe(target48k)); + + HRESULT mfHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (FAILED(mfHr) && mfHr != RPC_E_CHANGED_MODE) { + expect("mf-startup", false, "CoInitializeEx failed"); + } else { + mfHr = MFStartup(MF_VERSION); + expect("mf-startup", SUCCEEDED(mfHr), "MFStartup hr=" + std::to_string(static_cast(mfHr))); + if (SUCCEEDED(mfHr)) { + const HRESULT rejectHr = trySetAacPcmRate(96000); + char rejectHex[16]{}; + sprintf_s(rejectHex, "0x%08lx", static_cast(rejectHr)); + std::cout << "MF_RAW mf-reject-96000 hr=" << rejectHex << "\n"; + expect( + "mf-reject-96000", + FAILED(rejectHr) && static_cast(rejectHr) == 0xc00d36b4ul, + std::string("want 0xc00d36b4 got ") + rejectHex); + const HRESULT acceptHr = trySetAacPcmRate(48000); + char acceptHex[16]{}; + sprintf_s(acceptHex, "0x%08lx", static_cast(acceptHr)); + std::cout << "MF_RAW mf-accept-48000 hr=" << acceptHex << "\n"; + expect("mf-accept-48000", SUCCEEDED(acceptHr), std::string("hr=") + acceptHex); + + const HRESULT rejectAvHr = trySetAacPcmRateWithVideo(96000); + char rejectAvHex[16]{}; + sprintf_s(rejectAvHex, "0x%08lx", static_cast(rejectAvHr)); + std::cout << "MF_RAW mf-reject-96000-with-video hr=" << rejectAvHex << "\n"; + expect( + "mf-reject-96000-with-video", + FAILED(rejectAvHr), + std::string("want fail got ") + rejectAvHex); + const HRESULT acceptAvHr = trySetAacPcmRateWithVideo(48000); + char acceptAvHex[16]{}; + sprintf_s(acceptAvHex, "0x%08lx", static_cast(acceptAvHr)); + std::cout << "MF_RAW mf-accept-48000-with-video hr=" << acceptAvHex << "\n"; + expect( + "mf-accept-48000-with-video", + SUCCEEDED(acceptAvHr), + std::string("hr=") + acceptAvHex); + + Microsoft::WRL::ComPtr swAttr; + const HRESULT attrHr = MFCreateAttributes(&swAttr, 1); + if (FAILED(attrHr)) { + expect("mf-reject-96000-sw-attr", false, "MFCreateAttributes failed"); + } else { + swAttr->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, FALSE); + const HRESULT rejectSwHr = trySetAacPcmRate(96000, swAttr.Get()); + char rejectSwHex[16]{}; + sprintf_s(rejectSwHex, "0x%08lx", static_cast(rejectSwHr)); + std::cout << "MF_RAW mf-reject-96000-sw-attr hr=" << rejectSwHex << "\n"; + expect( + "mf-reject-96000-sw-attr", + FAILED(rejectSwHr), + std::string("want fail got ") + rejectSwHex); + const HRESULT acceptSwHr = trySetAacPcmRate(48000, swAttr.Get()); + char acceptSwHex[16]{}; + sprintf_s(acceptSwHex, "0x%08lx", static_cast(acceptSwHr)); + std::cout << "MF_RAW mf-accept-48000-sw-attr hr=" << acceptSwHex << "\n"; + expect( + "mf-accept-48000-sw-attr", + SUCCEEDED(acceptSwHr), + std::string("hr=") + acceptSwHex); + } + MFShutdown(); + } + } + + std::cout << "ran " << g_ran << " tests\n"; + if (g_failed != 0) { + std::cout << g_failed << " failed\n"; + return 1; + } + return 0; +} diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index a9e21d45c..4f795257c 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -810,17 +810,43 @@ int main(int argc, char* argv[]) { << jsonEscape(wideToUtf8(microphoneCapture.selectedDeviceName())) << "\""; } std::cout << "}" << std::endl; - encoderAudioFormat = makeAacCompatibleAudioFormat(*audioFormat); + AudioInputFormat sourceForEncoder = *audioFormat; + const int forcedAacSourceRate = readEnvInt("OPENSCREEN_WGC_FORCE_AAC_SOURCE_RATE", 0); + if (forcedAacSourceRate > 0) { + sourceForEncoder.sampleRate = static_cast(forcedAacSourceRate); + sourceForEncoder.avgBytesPerSec = + sourceForEncoder.sampleRate * sourceForEncoder.blockAlign; + } + if (readEnvInt("OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP", 0) == 1) { + encoderAudioFormat = sourceForEncoder; + encoderAudioFormat.subtype = MFAudioFormat_PCM; + encoderAudioFormat.channels = 2; + encoderAudioFormat.bitsPerSample = 16; + encoderAudioFormat.blockAlign = 4; + encoderAudioFormat.avgBytesPerSec = encoderAudioFormat.sampleRate * 4; + } else { + encoderAudioFormat = makeAacCompatibleAudioFormat(sourceForEncoder); + } + std::cout << "{\"event\":\"encoder-audio-format\",\"schemaVersion\":2,\"sampleRate\":" << encoderAudioFormat.sampleRate << ",\"channels\":" << encoderAudioFormat.channels << ",\"bitsPerSample\":" << encoderAudioFormat.bitsPerSample + << ",\"forcedSourceRate\":" << forcedAacSourceRate + << ",\"snapDisabled\":" + << (readEnvInt("OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP", 0) == 1 ? "true" : "false") + << ",\"aacRateProbe\":" + << (readEnvInt("OPENSCREEN_WGC_TEST_INJECT_AAC_RATE_PROBE", 0) == 1 ? "true" + : "false") << "}" << std::endl; } MFEncoderOptions encoderOptions{}; encoderOptions.preferSoftwareEncoder = config.preferSoftwareEncoder; encoderOptions.injectDefaultSinkWriterFailureOnce = injectDefaultSinkWriterFailureOnce; + encoderOptions.skipAacRateSnap = readEnvInt("OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP", 0) == 1; + encoderOptions.injectAacRateProbe = + readEnvInt("OPENSCREEN_WGC_TEST_INJECT_AAC_RATE_PROBE", 0) == 1; // OFF by default. The GPU path exists to dodge a Map() that wedges inside // the display driver on the machine in #252, and it demonstrably fixed // display and window capture there. It also broke recording outright for diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index 4130b1326..b4dabcc54 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -519,13 +519,15 @@ void setAudioFormat(IMFMediaType* type, UINT32 channels, UINT32 sampleRate, UINT // anything is built from it, not after. bool buildAacOutputType( const AudioInputFormat& audioFormat, + bool skipAacRateSnap, Microsoft::WRL::ComPtr& outputType) { if (audioFormat.sampleRate == 0 || audioFormat.channels == 0 || audioFormat.blockAlign == 0) { std::cerr << "ERROR: Invalid audio input format" << std::endl; return false; } - const AudioInputFormat encoderFormat = makeAacCompatibleAudioFormat(audioFormat); + const AudioInputFormat encoderFormat = + skipAacRateSnap ? audioFormat : makeAacCompatibleAudioFormat(audioFormat); const UINT32 aacBytesPerSecond = 24'000; if (!succeeded(MFCreateMediaType(&outputType), "MFCreateMediaType(audio output)")) { @@ -795,7 +797,7 @@ bool MFEncoder::initialize( // a construction argument. Null when the recording has no audio, which // is both the common case and a documented one for that call. Microsoft::WRL::ComPtr audioOutputType; - if (audioFormat && !buildAacOutputType(*audioFormat, audioOutputType)) { + if (audioFormat && !buildAacOutputType(*audioFormat, options.skipAacRateSnap, audioOutputType)) { return false; } @@ -859,7 +861,7 @@ bool MFEncoder::initialize( } } - if (audioFormat && !configureAudioStream(*audioFormat)) { + if (audioFormat && !configureAudioStream(*audioFormat, options)) { return false; } @@ -946,12 +948,75 @@ bool MFEncoder::initialize( // that used to sit between the two -- now happens before the sink writer // exists. What is left is the input type, which is the same on both containers // and is set on a stream index the caller has already resolved. -bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat) { +HRESULT probeAacPcmRate(UINT32 sampleRate) { + wchar_t dir[MAX_PATH]{}; + GetTempPathW(MAX_PATH, dir); + const std::wstring path = std::wstring(dir) + L"openscreen-wgc-aac-rate-probe-" + + std::to_wstring(GetCurrentProcessId()) + L".mp4"; + DeleteFileW(path.c_str()); + + Microsoft::WRL::ComPtr writer; + HRESULT hr = MFCreateSinkWriterFromURL(path.c_str(), nullptr, nullptr, &writer); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr outputType; + hr = MFCreateMediaType(&outputType); + if (FAILED(hr)) { + DeleteFileW(path.c_str()); + return hr; + } + outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + outputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_AAC); + setAudioFormat(outputType.Get(), 2, sampleRate, 16); + outputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, 24'000); + outputType->SetUINT32(MF_MT_AAC_PAYLOAD_TYPE, 0); + + DWORD streamIndex = 0; + hr = writer->AddStream(outputType.Get(), &streamIndex); + if (FAILED(hr)) { + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; + } + + Microsoft::WRL::ComPtr inputType; + hr = MFCreateMediaType(&inputType); + if (FAILED(hr)) { + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; + } + inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + inputType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); + setAudioFormat(inputType.Get(), 2, sampleRate, 16); + inputType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, 4); + inputType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, sampleRate * 4); + inputType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + hr = writer->SetInputMediaType(streamIndex, inputType.Get(), nullptr); + writer.Reset(); + DeleteFileW(path.c_str()); + return hr; +} + +bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat, const MFEncoderOptions& options) { if (!sinkWriter_) { return false; } - const AudioInputFormat encoderFormat = makeAacCompatibleAudioFormat(audioFormat); + const AudioInputFormat encoderFormat = + options.skipAacRateSnap ? audioFormat : makeAacCompatibleAudioFormat(audioFormat); + + if (options.injectAacRateProbe) { + const HRESULT probeHr = probeAacPcmRate(encoderFormat.sampleRate); + std::cerr << "TEST-ONLY: AAC rate probe sampleRate=" << encoderFormat.sampleRate + << " hr=0x" << std::hex << probeHr << std::dec << std::endl; + if (!succeeded(probeHr, "SetInputMediaType(audio)")) { + return false; + } + } Microsoft::WRL::ComPtr inputType; if (!succeeded(MFCreateMediaType(&inputType), "MFCreateMediaType(audio input)")) { diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index 19fac7004..f7176c75d 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -30,6 +30,17 @@ struct AudioInputFormat { struct MFEncoderOptions { bool preferSoftwareEncoder = false; bool injectDefaultSinkWriterFailureOnce = false; + // Test-only. Keep the sample rate passed into initialize() instead of + // running makeAacCompatibleAudioFormat again. The DISABLE_AAC_RATE_SNAP env + // used to change only the JSON log: buildAacOutputType / configureAudioStream + // snapped 96 kHz back to 48 kHz before SetInputMediaType, so the fail path + // was unreachable. + bool skipAacRateSnap = false; + // Test-only. Before the production sink writer sees the audio type, run + // MFCreateSinkWriterFromURL + SetInputMediaType on the encoder rate. Illegal + // AAC rates fail that probe with MF_E_INVALIDMEDIATYPE (0xc00d36b4). Does + // not invent the HRESULT — it calls the API. + bool injectAacRateProbe = false; // A request, never a requirement. Every step of the GPU path degrades to // the CPU readback rather than failing the recording, so a machine without // a hardware H.264 encoder, without NV12 video-processor output, or with a @@ -175,7 +186,7 @@ class MFEncoder { // built before the sink writer exists (buildAacOutputType in the .cpp), // because MFCreateFMPEG4MediaSink takes both output types at construction: // a fragmented sink has all its streams before anything can be added to it. - bool configureAudioStream(const AudioInputFormat& audioFormat); + bool configureAudioStream(const AudioInputFormat& audioFormat, const MFEncoderOptions& options); void releaseSinkWriter(); Microsoft::WRL::ComPtr sinkWriter_; diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs index 063d81b30..cfc7d057a 100644 --- a/scripts/build-windows-wgc-helper.mjs +++ b/scripts/build-windows-wgc-helper.mjs @@ -95,3 +95,10 @@ console.log(`Built ${outputPath}`); console.log(`Copied ${distributablePath}`); console.log(`Built ${cursorSamplerOutputPath}`); console.log(`Copied ${cursorSamplerDistributablePath}`); + +const audioUtilsTestPath = path.join(BUILD_DIR, "audio_sample_utils_test.exe"); +if (!fs.existsSync(audioUtilsTestPath)) { + throw new Error(`WGC helper build completed but ${audioUtilsTestPath} was not found.`); +} +await run(audioUtilsTestPath, [], { cwd: BUILD_DIR }); +console.log(`Passed ${audioUtilsTestPath}`); From ff60bdb2401728f18c4d0d0aa763a1b70feeb79f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E7=BA=A7?= =?UTF-8?q?=E5=96=9C=E6=AC=A2=E4=BD=A0=E7=9A=84=E8=BE=BE=E5=A6=AE=E5=A8=85?= <176143450+My-Denia@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:21:43 +0800 Subject: [PATCH 2/4] fix: anti-alias AAC downsample and ungate native build from MF probes --- .../wgc-capture/src/audio_sample_utils.cpp | 55 +++++-- .../src/audio_sample_utils_test.cpp | 152 +++++++++--------- electron/native/wgc-capture/src/mf_encoder.h | 2 +- scripts/build-windows-wgc-helper.mjs | 3 + .../testing/manual-e2e-checklist.md | 1 + 5 files changed, 131 insertions(+), 82 deletions(-) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp index 26fb57df5..f62008354 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -207,6 +207,40 @@ void convertAudioWithGain( return; } + // Integer-factor downsample (96 kHz / 192 kHz -> 48 kHz): average each + // group of source frames instead of picking one. Nearest-neighbour + // decimation aliases content above the new Nyquist into the recording. + if (sourceFormat.sampleRate > targetFormat.sampleRate && + sourceFormat.sampleRate % targetFormat.sampleRate == 0) { + const UINT32 factor = sourceFormat.sampleRate / targetFormat.sampleRate; + const size_t targetFrames = sourceFrames / factor; + if (targetFrames == 0) { + destination.clear(); + return; + } + destination.assign(targetFrames * targetFormat.blockAlign, 0); + for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { + for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { + double sum = 0.0; + for (UINT32 tap = 0; tap < factor; ++tap) { + sum += readMappedChannel( + source, + sourceFormat, + targetFrame * factor + tap, + channel, + targetFormat.channels); + } + writeSampleFromDouble( + destination.data(), + targetFormat, + targetFrame, + channel, + (sum / static_cast(factor)) * gain); + } + } + return; + } + const double rateRatio = static_cast(targetFormat.sampleRate) / static_cast(sourceFormat.sampleRate); const size_t targetFrames = std::max(1, static_cast(std::llround(sourceFrames * rateRatio))); @@ -214,17 +248,20 @@ void convertAudioWithGain( for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { const double sourcePosition = static_cast(targetFrame) / rateRatio; - const size_t sourceFrame = std::min( - sourceFrames - 1, - static_cast(std::llround(sourcePosition))); + const size_t sourceFrame = std::min(sourceFrames - 1, static_cast(sourcePosition)); + const size_t nextFrame = std::min(sourceFrames - 1, sourceFrame + 1); + const double frac = sourcePosition - static_cast(sourceFrame); for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { - const double sample = readMappedChannel( - source, - sourceFormat, - sourceFrame, + const double a = readMappedChannel( + source, sourceFormat, sourceFrame, channel, targetFormat.channels); + const double b = readMappedChannel( + source, sourceFormat, nextFrame, channel, targetFormat.channels); + writeSampleFromDouble( + destination.data(), + targetFormat, + targetFrame, channel, - targetFormat.channels); - writeSampleFromDouble(destination.data(), targetFormat, targetFrame, channel, sample * gain); + (a + (b - a) * frac) * gain); } } } diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp index 81f5cfb78..0dc7e8c00 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp @@ -7,8 +7,10 @@ #include #include +#include #include #include +#include #include namespace { @@ -41,6 +43,22 @@ void expect(const char* name, bool ok, const std::string& detail) { std::cout << "FAIL " << name << " " << detail << "\n"; } +void skip(const char* name, const std::string& reason) { + std::cout << "SKIP " << name << " " << reason << "\n"; +} + +struct TempMp4 { + std::wstring path; + explicit TempMp4(std::wstring p) : path(std::move(p)) { + DeleteFileW(path.c_str()); + } + ~TempMp4() { + DeleteFileW(path.c_str()); + } + TempMp4(const TempMp4&) = delete; + TempMp4& operator=(const TempMp4&) = delete; +}; + std::string describe(const AudioInputFormat& format) { return "sampleRate=" + std::to_string(format.sampleRate) + " channels=" + std::to_string(format.channels) + @@ -55,11 +73,10 @@ std::wstring tempMp4Path() { } HRESULT trySetAacPcmRate(UINT32 sampleRate, IMFAttributes* attributes = nullptr) { - const std::wstring path = tempMp4Path(); - DeleteFileW(path.c_str()); + TempMp4 tmp(tempMp4Path()); Microsoft::WRL::ComPtr writer; - HRESULT hr = MFCreateSinkWriterFromURL(path.c_str(), nullptr, attributes, &writer); + HRESULT hr = MFCreateSinkWriterFromURL(tmp.path.c_str(), nullptr, attributes, &writer); if (FAILED(hr)) { return hr; } @@ -80,14 +97,12 @@ HRESULT trySetAacPcmRate(UINT32 sampleRate, IMFAttributes* attributes = nullptr) DWORD streamIndex = 0; hr = writer->AddStream(outputType.Get(), &streamIndex); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } Microsoft::WRL::ComPtr inputType; hr = MFCreateMediaType(&inputType); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); @@ -101,7 +116,6 @@ HRESULT trySetAacPcmRate(UINT32 sampleRate, IMFAttributes* attributes = nullptr) hr = writer->SetInputMediaType(streamIndex, inputType.Get(), nullptr); writer.Reset(); - DeleteFileW(path.c_str()); return hr; } @@ -109,11 +123,10 @@ HRESULT trySetAacPcmRate(UINT32 sampleRate, IMFAttributes* attributes = nullptr) // topology. Distinguishes "96 kHz AAC is illegal" from "audio-only MP4 sink // writer refuses this type". HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { - const std::wstring path = tempMp4Path(); - DeleteFileW(path.c_str()); + TempMp4 tmp(tempMp4Path()); Microsoft::WRL::ComPtr writer; - HRESULT hr = MFCreateSinkWriterFromURL(path.c_str(), nullptr, nullptr, &writer); + HRESULT hr = MFCreateSinkWriterFromURL(tmp.path.c_str(), nullptr, nullptr, &writer); if (FAILED(hr)) { return hr; } @@ -134,14 +147,12 @@ HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { DWORD videoIndex = 0; hr = writer->AddStream(videoOut.Get(), &videoIndex); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } Microsoft::WRL::ComPtr audioOut; hr = MFCreateMediaType(&audioOut); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } audioOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); @@ -155,14 +166,12 @@ HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { DWORD audioIndex = 0; hr = writer->AddStream(audioOut.Get(), &audioIndex); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } Microsoft::WRL::ComPtr videoIn; hr = MFCreateMediaType(&videoIn); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } videoIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); @@ -174,14 +183,12 @@ HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { MFSetAttributeRatio(videoIn.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); hr = writer->SetInputMediaType(videoIndex, videoIn.Get(), nullptr); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } Microsoft::WRL::ComPtr audioIn; hr = MFCreateMediaType(&audioIn); if (FAILED(hr)) { - DeleteFileW(path.c_str()); return hr; } audioIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); @@ -194,7 +201,6 @@ HRESULT trySetAacPcmRateWithVideo(UINT32 sampleRate) { audioIn->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); hr = writer->SetInputMediaType(audioIndex, audioIn.Get(), nullptr); writer.Reset(); - DeleteFileW(path.c_str()); return hr; } @@ -251,66 +257,68 @@ int main() { target48k.sampleRate == 48000 && frameCountOk, "frames=" + std::to_string(convertedFrames) + " " + describe(target48k)); + // 96 kHz Nyquist square (+/- full scale) must not survive 2:1 as a tone. + std::vector nyquist(8 * source96k.blockAlign, 0); + auto* nyquistSamples = reinterpret_cast(nyquist.data()); + for (size_t frame = 0; frame < 8; frame += 1) { + const int16_t v = (frame % 2 == 0) ? 32767 : -32767; + nyquistSamples[frame * 2] = v; + nyquistSamples[frame * 2 + 1] = v; + } + std::vector nyquistOut; + convertAudioWithGain(nyquist.data(), static_cast(nyquist.size()), source96k, target48k, 1.0, nyquistOut); + const auto* down = reinterpret_cast(nyquistOut.data()); + const size_t downFrames = nyquistOut.size() / target48k.blockAlign; + bool folded = downFrames == 4; + for (size_t i = 0; folded && i < downFrames * 2; i += 1) { + folded = std::abs(static_cast(down[i])) <= 1; + } + expect("resample-96k-nyquist-box", folded, "frames=" + std::to_string(downFrames)); + HRESULT mfHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); if (FAILED(mfHr) && mfHr != RPC_E_CHANGED_MODE) { - expect("mf-startup", false, "CoInitializeEx failed"); + skip("mf-startup", "CoInitializeEx failed — no Media Foundation on this host"); } else { mfHr = MFStartup(MF_VERSION); - expect("mf-startup", SUCCEEDED(mfHr), "MFStartup hr=" + std::to_string(static_cast(mfHr))); - if (SUCCEEDED(mfHr)) { - const HRESULT rejectHr = trySetAacPcmRate(96000); - char rejectHex[16]{}; - sprintf_s(rejectHex, "0x%08lx", static_cast(rejectHr)); - std::cout << "MF_RAW mf-reject-96000 hr=" << rejectHex << "\n"; - expect( - "mf-reject-96000", - FAILED(rejectHr) && static_cast(rejectHr) == 0xc00d36b4ul, - std::string("want 0xc00d36b4 got ") + rejectHex); - const HRESULT acceptHr = trySetAacPcmRate(48000); - char acceptHex[16]{}; - sprintf_s(acceptHex, "0x%08lx", static_cast(acceptHr)); - std::cout << "MF_RAW mf-accept-48000 hr=" << acceptHex << "\n"; - expect("mf-accept-48000", SUCCEEDED(acceptHr), std::string("hr=") + acceptHex); - - const HRESULT rejectAvHr = trySetAacPcmRateWithVideo(96000); - char rejectAvHex[16]{}; - sprintf_s(rejectAvHex, "0x%08lx", static_cast(rejectAvHr)); - std::cout << "MF_RAW mf-reject-96000-with-video hr=" << rejectAvHex << "\n"; - expect( - "mf-reject-96000-with-video", - FAILED(rejectAvHr), - std::string("want fail got ") + rejectAvHex); - const HRESULT acceptAvHr = trySetAacPcmRateWithVideo(48000); - char acceptAvHex[16]{}; - sprintf_s(acceptAvHex, "0x%08lx", static_cast(acceptAvHr)); - std::cout << "MF_RAW mf-accept-48000-with-video hr=" << acceptAvHex << "\n"; - expect( - "mf-accept-48000-with-video", - SUCCEEDED(acceptAvHr), - std::string("hr=") + acceptAvHex); - - Microsoft::WRL::ComPtr swAttr; - const HRESULT attrHr = MFCreateAttributes(&swAttr, 1); - if (FAILED(attrHr)) { - expect("mf-reject-96000-sw-attr", false, "MFCreateAttributes failed"); - } else { - swAttr->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, FALSE); - const HRESULT rejectSwHr = trySetAacPcmRate(96000, swAttr.Get()); - char rejectSwHex[16]{}; - sprintf_s(rejectSwHex, "0x%08lx", static_cast(rejectSwHr)); - std::cout << "MF_RAW mf-reject-96000-sw-attr hr=" << rejectSwHex << "\n"; - expect( - "mf-reject-96000-sw-attr", - FAILED(rejectSwHr), - std::string("want fail got ") + rejectSwHex); - const HRESULT acceptSwHr = trySetAacPcmRate(48000, swAttr.Get()); - char acceptSwHex[16]{}; - sprintf_s(acceptSwHex, "0x%08lx", static_cast(acceptSwHr)); - std::cout << "MF_RAW mf-accept-48000-sw-attr hr=" << acceptSwHex << "\n"; - expect( - "mf-accept-48000-sw-attr", - SUCCEEDED(acceptSwHr), - std::string("hr=") + acceptSwHex); + if (FAILED(mfHr)) { + skip("mf-startup", "MFStartup hr=" + std::to_string(static_cast(mfHr))); + } else { + expect("mf-startup", true, ""); + const auto runReject = [](const char* name, HRESULT hr) { + char hex[16]{}; + sprintf_s(hex, "0x%08lx", static_cast(hr)); + std::cout << "MF_RAW " << name << " hr=" << hex << "\n"; + if (FAILED(hr) && static_cast(hr) == 0xc00d36b4ul) { + expect(name, true, ""); + } else if (SUCCEEDED(hr)) { + skip(name, "host AAC accepts 96 kHz"); + } else { + skip(name, std::string("host cannot probe this rate hr=") + hex); + } + }; + const auto runAccept = [](const char* name, HRESULT hr) { + char hex[16]{}; + sprintf_s(hex, "0x%08lx", static_cast(hr)); + std::cout << "MF_RAW " << name << " hr=" << hex << "\n"; + if (SUCCEEDED(hr)) { + expect(name, true, ""); + return true; + } + skip(name, std::string("host has no AAC encoder hr=") + hex); + return false; + }; + runReject("mf-reject-96000", trySetAacPcmRate(96000)); + if (runAccept("mf-accept-48000", trySetAacPcmRate(48000))) { + runReject("mf-reject-96000-with-video", trySetAacPcmRateWithVideo(96000)); + runAccept("mf-accept-48000-with-video", trySetAacPcmRateWithVideo(48000)); + Microsoft::WRL::ComPtr swAttr; + if (FAILED(MFCreateAttributes(&swAttr, 1))) { + skip("mf-reject-96000-sw-attr", "MFCreateAttributes failed"); + } else { + swAttr->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, FALSE); + runReject("mf-reject-96000-sw-attr", trySetAacPcmRate(96000, swAttr.Get())); + runAccept("mf-accept-48000-sw-attr", trySetAacPcmRate(48000, swAttr.Get())); + } } MFShutdown(); } diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index f7176c75d..8d1d6ae6e 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -31,7 +31,7 @@ struct MFEncoderOptions { bool preferSoftwareEncoder = false; bool injectDefaultSinkWriterFailureOnce = false; // Test-only. Keep the sample rate passed into initialize() instead of - // running makeAacCompatibleAudioFormat again. The DISABLE_AAC_RATE_SNAP env + // running makeAacCompatibleAudioFormat again. OPENSCREEN_WGC_DISABLE_AAC_RATE_SNAP // used to change only the JSON log: buildAacOutputType / configureAudioStream // snapped 96 kHz back to 48 kHz before SetInputMediaType, so the fail path // was unreachable. diff --git a/scripts/build-windows-wgc-helper.mjs b/scripts/build-windows-wgc-helper.mjs index cfc7d057a..8d4cfc018 100644 --- a/scripts/build-windows-wgc-helper.mjs +++ b/scripts/build-windows-wgc-helper.mjs @@ -100,5 +100,8 @@ const audioUtilsTestPath = path.join(BUILD_DIR, "audio_sample_utils_test.exe"); if (!fs.existsSync(audioUtilsTestPath)) { throw new Error(`WGC helper build completed but ${audioUtilsTestPath} was not found.`); } +// Snap/resample unit tests must pass. Media Foundation AAC probes skip on +// hosts without the stock encoder (Windows N/KN, Server without Media Feature +// Pack) instead of failing this packaging command. await run(audioUtilsTestPath, [], { cwd: BUILD_DIR }); console.log(`Passed ${audioUtilsTestPath}`); diff --git a/technical-documentation/testing/manual-e2e-checklist.md b/technical-documentation/testing/manual-e2e-checklist.md index 8609f0a14..02a3b542c 100644 --- a/technical-documentation/testing/manual-e2e-checklist.md +++ b/technical-documentation/testing/manual-e2e-checklist.md @@ -448,6 +448,7 @@ The agent may only call the fixed tool set in [ai-agent.md](../architecture/ai-a | 2026-07-31 | dev build, `claude/e2e-tests-v1-8-0-474894` (e9578f09) | macOS 26.5, M1 | Partial — 1 defect | Ran launch/HUD, media, modifier anchoring, and export. **Defect: a dangling asset blanks the preview.** Modifier anchoring across a reorder verified in preview and in the exported frames. macOS export produced 1280×720 h264 + AAC at ~2× realtime. Chat sections skipped: no AI provider configured. HUD drag not runnable under computer-use (drop point is the desktop). | | 2026-08-13 | installed `v1.9.5-rc.1` | Windows 11 26200, 1920×1080 @ 100% | Partial — 1 defect | Ran launch/HUD, source selection, recording, stop, editor open. Fragmented MP4 confirmed on the shipped artifact: 48 `moof`+`mdat` pairs over 47.6s, `mvex` present, `mfra` on clean stop. **Defect: a recording that survives a helper kill is thrown away by the app** — killing `wgc-capture.exe` mid-recording leaves a fully decodable 41s file (2460 packets, `ffmpeg -f null -` exit 0) with no `.session.json` and no `.cursor.json`, and stop answers "The recording could not be saved". Fixed in #363, re-verified end to end. Truncation ablation at 60%: plain MP4 unreadable, fragmented plays 29s. **A dev build cannot test any of this** — the prebuilt worktree helper predated the change and silently ran the old path. Editor/export/chat sections not run. | | 2026-08-14 | `release/v1.9.5` @ `b1b81de5` (rc.2 candidate: dev TS + the CI-built rc.1 native payload, which is byte-identical since no native source changed) | Windows 11 26200, 1920×1080 @ 100% | Pass — no defect | Regression net across the 65 commits since **v1.9.2**, not just the rc.2 delta. Four recordings. Every one a fragmented MP4 (`mvex` + ~1 `moof`/s, `mfra` only on a clean stop). GPU DXGI path still correctly opt-in (`videoInput: cpu-rgb32`) — the #336 regression has not crept back. No capture-pacing drift: HUD `00:59` → 60.067s at 60/1. Waveform correct in both directions: absent with no audio track, rendered with one. Audio muxes into the fragmented container (AAC 48k stereo) with 15 ms A/V drift, under one frame. Compositor renders and exports with no camera declared. **#366**: reopening returns to the saved project with its settings (Blur BG on, padding 9%) and mints no second project — 167→168 across a whole new recording. **#363**: helper killed mid-recording → editor opens on the recovered take (46 `moof`, no `mfra`), all three sidecars written, imported once. Export MP4 1080p60 **from that recovered take**: 46.0s / 2760 packets, decodes clean, duration matches the source exactly. Tray refocus works. NOT covered: DPI scaling — **not re-run here, already validated when `60bb6d7c` / `71cc88d6` landed**; note that the display scale is a setting, so "this machine is at 100%" is never a reason a DPI bug cannot be tested (flip it to 150%, ~2 min). Also not covered: webcam PiP and the export webcam fixes, microphone, GIF, macOS/Linux, AI sections, packaging. | +| 2026-09-02 | worktree `fix/wgc-aac-sample-rate` (rebuilt `wgc-capture.exe`) | Windows 11 | Partial — helper-level only | Not a capture-to-export HUD pass. Media Foundation `SetInputMediaType` for 96 kHz PCM into AAC returns `0xc00d36b4`; 48 kHz succeeds. Live helper with snap on: forced 96 kHz encoder source reports 48 kHz and `recording-started`. Live helper with snap off: initialize fails at `SetInputMediaType(audio)` with `0xc00d36b4` (hardware, software, and plain MP4 retries). HUD, editor, export, webcam, and tray not run. | | 2026-08-14 | installed `v1.9.5-rc.1`, macOS Apple Silicon DMG (CI-built, Developer ID signed). **rc.2 is not published** — only rc.1 exists on Releases; no native source changed between `v1.9.5-rc.1` and `origin/release/v1.9.5`, so this artifact already carries the rc.2 native payload, but #366 (cross-platform TS) is absent from it | macOS 26.5 (25F71), M1, 1920×1080 @ 2× | **Fail — 1 blocker** | **The plan's assertion-1 criterion does not hold on macOS, in both directions.** On a clean stop `AVAssetWriter.finishWriting()` collapses the fragments into a normal movie: `ftyp mdat moov`, `mvex` ABSENT, 0 `moof`, no `mfra` (45 s / 44.4 MB run). That is exactly the shape the plan calls the headline failure — and the pre-`a6795d23` control recording (2026-08-10) has the *same* shape — so **a clean-stop box walk cannot distinguish fragmented from plain on macOS; only the kill test can.** Fragmenting *is* active: the takes whose writer died mid-fragment retain `mvex` + ~1 `moof` per second of media (shipped-build writer-failure samples: 35 `moof`/36.0 s, 14/15.0 s, 3/4.0 s; plus 18 on a surviving-helper kill). The one kill on the shipped build is the exception that proves the scope — capture had already stalled ~12 s before the kill, so it carries `mvex` but **0 `moof`** and only 1.0 s. No macOS file, clean or killed, ever carried `mfra`. **Blocker: every app-driven recording truncates, then the app discards it.** (Root cause and fix reported in #375 — the fragments carry a negative composition offset in a version 0 `trun`, where ISO/IEC 14496-12 8.8.8.2 defines the field as unsigned, because frame reordering was left on; `AVVideoAllowFrameReorderingKey: false` clears it and restores the crash-resilience the fragmenting was for. Verified at helper level there; **this rc.1 run only reproduced the failure and validated nothing about the fix**. Re-run this section against a CI build carrying #375 before rc.2 ships.) 3/3 takes stopped writing early while the HUD kept counting — media 4.0 s / 36.0 s / 15.0 s against HUD `02:02` / `01:30` / `01:04`. Helper emits `{"event":"error","code":"writer-failed"}`; main log `AVFoundationErrorDomain Code=-11800 … (-16341)`. Stop then hangs ~30 s on "Saving…" and drops the take: no `.session.json`, no `.cursor.json`, no editor. The app *does* surface the raw error in a toast (confirmed by hand on the same machine at 13:28–13:35 — my automated runs screenshotted after it auto-dismissed, so an earlier draft of this row wrongly said there was none). 44,561,966 / 328,337,979 / 139,631,607 / 17,187,009 bytes decodable and thrown away (147 GB free — not disk). Reproduced standalone with the shipped helper at 1080p30/8 Mbps, 2/2 (~9 s, ~5 s), so it is not confined to the app's 4K60 path — but do not read that as load-independent: append rate demonstrably modulates how reliably it bites (#375 measures it reliable at ~57 fps and intermittent at 30 fps). **Reproduced by hand, no automation involved**, on six takes recording a YouTube page — and those six separate the trigger cleanly: **system audio ON → 3/3 died at ~1.0 s and minted 0 projects; system audio OFF → 3/3 survived (3.3 s, 7.4 s, 25.0 s) and minted 1 project each.** **Audio is not the condition, only an accelerant** — a controlled run with system audio off *and not one screenshot taken during the capture* (the screenshot layer hides non-allowlisted windows, so it was the last confound worth eliminating) died the same way: 8.008 s of video, 79,004,330 bytes then flat for 76 s with the helper still alive, 7 `moof`, 0 sidecars, 0 projects, same `-11800`/`-16341`. What audio changes is the window: with a track it is ~1 s, without one ~4–40 s. That reconciles the by-hand takes with mine — a take short enough to stop before the writer dies is clean, which is why 3.3 s and 7.4 s survived and 8.0 s did not, and why the 25.0 s one minted a project while still carrying `mvex` (never cleanly finalised). **Turning audio off is therefore not a safe workaround.** Untested here: microphone — this Mac has no input device, and whether a mic track triggers the same path is an inference, not a measurement. **Helper A/B narrows the with-audio path to the fragmentation line**: helper built twice from source identical to the rc.1 tag, differing only by `writer.movieFragmentInterval` (701 vs 700 lines) — with system audio at 1080p30, WITH the line `writer-failed` 2/2 (2.0 s, 1.0 s), WITHOUT it clean `recording-stopped` 3/3 (40.6 s, 37.9 s, 37.6 s). **Read those counts as a sample, not a law**: a later rebuild of the with-the-line arm survived 22.2 s at the same settings, so the failure is probabilistic and rate-dependent, and the byte-level evidence in #375 is what actually carries the case. The video-only local-vs-shipped gap (local survived 45 s, shipped failed 5/5) is explained by the same variable rather than by the released artifact — the shipped runs encoded at 56.6 fps against 29 fps locally. **Kill test** is confounded on the shipped build (capture already dead before the kill): 17.19 MB → only 1.0 s / 56 packets, 0 `moof`. On a helper that does not fail, a mid-write kill leaves 18 `moof`, decodes clean (`ffmpeg -v error -f null -` exit 0, 1373 packets) and no `mfra` — the shape the plan expects. **#363 gap confirmed, and on macOS it fires with no kill at all**: `writer-failed` alone loses the take; there is no app-side recovery. **Audio**: AAC 48 kHz stereo muxes into the fragmented container, video start `0.000000` vs audio `0.014479` → 14.5 ms drift, under one frame at 30 fps (measured on the 2.0 s written before the writer died). **Compositor + export pass**: preview renders with no camera declared; export MP4 1080p60 H.264+AAC via `h264_videotoolbox (zero-copy VT)`, 5,726,865 bytes, 318 packets, decodes clean, duration matches to within 7 ms — source 26.713 s minus trims 19.910 + 1.513 = 5.290 s expected vs 5.283 s measured, under one frame at 60 fps. **#366 not runnable as specified** (absent from rc.1, rc.2 unpublished, and record→editor never completes); adjacent behaviour measured on an existing project — close+reopen kept 19→19 projects, exactly ONE project references the recording, and Blur BG / padding survived (`showBlur=true`, `padding=16`). NOT covered: Windows-only DPI and wgc-capture, GIF, AI sections, packaging (per plan); webcam PiP and microphone — this Mac has neither (Device settings reports "No microphone found" / "No camera found"). | | 2026-08-22 | installed `v1.10.0-rc.3` — CI-built NSIS artifact from build run 32582966489 (`openscreen-windows`), App menu → About reports `1.10.0-rc.3`, native payload complete and uniformly stamped (19 files in `resources/electron/native/bin/win32-x64`, all `17:59:10`, so helper + compositor addon + av\* DLLs are one matched CI set) | Windows 11 26200, 1920×1080 @ 100% | **Pass — 2 minor defects** | **Pause works, and the measurement that says so is the wall clock.** `createdAt` 20:25:52.208 against a file finalised at 20:30:56.754 is 304.55 s elapsed for a **286.333 s** file — **18.21 s shorter, exactly the paused interval**, so capture was genuinely suspended. The HUD timer froze at `03:58` across two reads 7 s apart with the indicator amber, and resume was clean (`04:01` → `04:08` over 7 s, no time lost). An earlier draft of this row called this a blocking defect, on the strength of comparing the file duration against a timer read *before* the stop click; with tool round-trips of ~20 s that comparison is worthless, and the packet count offered as corroboration proves nothing either — a file is continuous 60 fps whether or not capture was ever suspended. Written down because the wrong version of this measurement is easy to repeat: compare against wall-clock elapsed, never against the last timer you happened to screenshot. **Capture is otherwise sound, on two takes.** 15.8 s: fragmented (`ftyp uuid pdin moov` then 16 `moof`/`mdat`, `mvex` present), `mfra` on the clean stop, 1920×1080 @ 60/1, 948 packets = 15.8 × 60, `ffmpeg -v error -f null -` exit 0, both sidecars written. 286.3 s: 287 `moof`, `mfra` present, 17,180 packets, decodes clean, `.cursor.json` 1.3 MB. No pacing drift and no dropped frames over 4 min 46. **Export passes and honours its settings**: 720p/30 requested from a 1080p60 source gave 1280×720, `avg_frame_rate` 85900/2863 = 30.004, 8590 packets matching the frame count the progress UI itself reported, duration 286.333 s identical to source, decodes clean, 124.5 MB, written to the path chosen in the native save dialog and reported back as "Saved to …". Composition verified by extracting a frame and reading it at full resolution (not from a preview screenshot): gradient background, content inset as a rounded card with a drop shadow, content aspect ≈1.76 against the 16:9 target, synthetic cursor drawn. Note the exporter adds a silent **AAC 48 kHz stereo** track even though no audio source was enabled. **Retracted: "the HUD language menu ignores `Escape`".** It does not — the maintainer confirms the key works by hand. **Claude Desktop swallows `Escape` before it reaches the app under test**, so a synthesised press proves nothing about the app, and `GetForegroundWindow()` returning the HUD does not rescue the inference: the key never left the driver. The companion observation (an outside click on the HUD's own drag handle did not dismiss the menu) is withdrawn with it, since the HUD's own chrome is not "outside" the popover in any meaningful sense. What *is* established is that the blur path shipped in this RC works: `54e12706 fix(hud): dismiss the HUD popovers when the window loses focus` dismissed the menu on a click to the desktop. **Rule for anyone driving keyboard checks from computer-use: `Escape` is unusable as evidence, and any negative keyboard result needs a by-hand confirmation before it goes in this table.** **Behaviour vs doc**: the record button is not disabled without a source — it opens the source selector. No recording starts, so the check's intent holds, but AGENTS.md still describes a disabled button with a "Please select a source to record" tooltip, and that is why no tooltip appears. **Passed**: single launch window, no startup crash; HUD visible under `OPENSCREEN_DISABLE_CONTENT_PROTECTION=1`; tray layout toggles horizontal↔vertical both ways; HUD drag follows the pointer without drift and stays at the drop point; language menu opens with its locale list; minimize hides the HUD without quitting (6 processes still alive); relaunching routes through the single-instance lock, restores the window and mints no duplicate; source selector opens, selecting a card enables Share, and the HUD label becomes the picked source (`Tout l'écran`); record → stop opens the editor with the asset, a timeline clip and a rendered preview; About reports the RC version. **Local transcription works, on GPU** — an earlier draft of this row reported it broken, which was wrong. Relaunching with stdout/stderr captured and importing a 15 s asset that carries an audio track settles it: `[whisper-stt] boot: model=…\whisper-ggml\ggml-small-q8_0.bin host=127.0.0.1 port=64720 threads=16`, `ggml_vulkan: 0 = NVIDIA GeForce RTX 4070 Ti`, `model loaded; backend=whispercpp-vulkan`, then `[stt] done on whispercpp-vulkan: 1 chunk(s), 15.0s audio in 0.1s (0.01 rtf, 106.8x real-time)`. The pane switched to "1 caption lines, derived live from the transcript". **The real (minor) defect is the error message**: on an asset with *no audio track* the captions pane says **"Failed to fetch"**, which reads as a network failure and sent this run hunting a broken STT server that was never involved — the pipeline simply has no audio to extract. It should say so. **Second minor find, from the same stderr**: `listProjects` cannot read three saved projects — one `ZodError` (`transcript.segments[0].endSec must be greater than or equal to startSec`, repeated across `segments`, `words` and `transcripts[0]`) and two `SyntaxError: Unexpected non-whitespace character after JSON`, i.e. truncated or double-written project files. They are skipped silently in the UI. **Caption anchoring — the rc.2→rc.3 delta — is present but its rendering was not measured.** The Position section carries exactly the model those commits describe: `Bottom`/`Top`, the note "Long captions grow upward — the bottom edge stays put", `Distance from bottom` defaulting to **1.5 %**, and Left/Center/Right. What could not be checked is where a caption actually lands, because the only transcript obtainable here came from a 300 Hz sine and yielded one line that never surfaced at any scrubbed position. **Closed out of band: the maintainer ran the caption sections by hand on a real spoken-audio recording and reports them correct**, which is the coverage this automated run could not supply and the last gap standing between this RC and a promote. Also confirmed from stderr: `[content-protection] OFF for the HUD window (OPENSCREEN_DISABLE_CONTENT_PROTECTION=1)`, so the flag does log its effect, and with the flag unset the HUD is correctly invisible to screenshots. **The consequence matters more than the cause: the eight caption anchoring/margin/inset cherry-picks that are the entire delta from rc.2 to rc.3 are NOT covered by this run.** **Not run**: restart and cancel actions; audio capture of any kind; webcam PiP; GIF; DPI scaling; HUD/notes exclusion from captured video with content protection ON (the whole session ran with it off, and the exported frame confirms the HUD *is* captured when it is off); regions, modifiers, timeline navigation, clip operations, persistence; macOS and Linux. **Environment limits that shaped this run, worth knowing before the next one.** `parsecd.exe` runs **elevated** and holds an invisible always-foreground window (`ParsecMinFrameRate16`); the moment OpenScreen loses focus every computer-use click is refused, and because the process is elevated UIPI makes granting Parsec useless — **tray-icon refocus could therefore not be tested at all**. Relaunching the app (single-instance raises it) is the way back. Dragging the HUD only works while every intermediate pointer position stays inside the HUD's own 904×698 mostly-transparent window; as soon as one lands on the desktop, the tier-"click" shell gate refuses the drag mid-gesture and leaves the button down — release it explicitly. Finally, the Microsoft Store package (`EtienneLescot.OpenScreen`, 1.9.6) **shadows the NSIS install in `request_access`**: every grant resolved to the Store bundle and the RC window stayed masked in screenshots while reporting success, until the Store package was removed. Screenshots do **not** interrupt a recording — that hypothesis was raised and disproved by running a 90 s capture with none taken and then taking one mid-capture with the helper surviving. | | 2026-08-23 | installed `v1.10.0-rc.3` (Developer ID, unmodified) run with `OPENSCREEN_SCK_CAPTURE_EXE` pointed at a helper built from this branch | macOS 26.6.2 (25G83), M1, 1728×1117 @ 2× | Pass — fixes a blocker | **Window capture section only.** Before: selecting any window in the source picker kills the helper the instant `start()` builds its filter — `Assertion failed: (did_initialize), function CGS_REQUIRE_INIT, file CGInitialization.c, line 44`, SIGABRT, `-[SCContentFilter initWithDesktopIndependentWindow:]` → `SLSGetDisplaysWithRect`. 6/6 attempts on the shipped rc.3, no file, no error surfaced in the UI (the HUD returns to idle as if nothing happened). Display capture is unaffected and always worked, which is why this went unnoticed: the two paths diverge at `makeCaptureTarget`, and only the window branch resolves a rect through SkyLight. After: record → 25.2s → stop → **editor opened on the take**, `recording-1787475175449.mp4` 12,559,123 bytes / 25.18s / 2674×1684, the MP4 and both sidecars written (`.cursor.json`, `.session.json`), one project minted, zero crash reports. Helper-level A/B on an identical request JSON isolates the change: shipped signed helper → assertion, no file; this branch's helper → `recording-started`/`recording-stopped`, 4.49s / 1336×840 decodable MP4. NOT covered: webcam PiP, microphone, system audio (all off for these runs), export, GIF, AI/transcript sections, Windows, Linux. Not covered by unit tests either — `Package.swift` scopes the Swift test target to what runs without a screen, a display server or a TCC grant, and this crash needs all three. | From 6abf9ad08f25b6124f2c32c7856affeae4f980e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E7=BA=A7?= =?UTF-8?q?=E5=96=9C=E6=AC=A2=E4=BD=A0=E7=9A=84=E8=BE=BE=E5=A6=AE=E5=A8=85?= <176143450+My-Denia@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:40:51 +0800 Subject: [PATCH 3/4] fix: keep sub-factor AAC packets on the interpolation path --- .../wgc-capture/src/audio_sample_utils.cpp | 42 +++++++++---------- .../src/audio_sample_utils_test.cpp | 20 +++++++++ 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp index f62008354..ec6ecdbd7 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -214,31 +214,31 @@ void convertAudioWithGain( sourceFormat.sampleRate % targetFormat.sampleRate == 0) { const UINT32 factor = sourceFormat.sampleRate / targetFormat.sampleRate; const size_t targetFrames = sourceFrames / factor; - if (targetFrames == 0) { - destination.clear(); - return; - } - destination.assign(targetFrames * targetFormat.blockAlign, 0); - for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { - for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { - double sum = 0.0; - for (UINT32 tap = 0; tap < factor; ++tap) { - sum += readMappedChannel( - source, - sourceFormat, - targetFrame * factor + tap, + if (targetFrames > 0) { + destination.assign(targetFrames * targetFormat.blockAlign, 0); + for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { + for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { + double sum = 0.0; + for (UINT32 tap = 0; tap < factor; ++tap) { + sum += readMappedChannel( + source, + sourceFormat, + targetFrame * factor + tap, + channel, + targetFormat.channels); + } + writeSampleFromDouble( + destination.data(), + targetFormat, + targetFrame, channel, - targetFormat.channels); + (sum / static_cast(factor)) * gain); } - writeSampleFromDouble( - destination.data(), - targetFormat, - targetFrame, - channel, - (sum / static_cast(factor)) * gain); } + return; } - return; + // Too few source frames for one averaged output frame (tiny WASAPI + // packets). Fall through to interpolation instead of dropping them. } const double rateRatio = static_cast(targetFormat.sampleRate) / diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp index 0dc7e8c00..b261a7d28 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -275,6 +276,25 @@ int main() { } expect("resample-96k-nyquist-box", folded, "frames=" + std::to_string(downFrames)); + std::vector shortPkt(source96k.blockAlign, 0); + auto* shortSamples = reinterpret_cast(shortPkt.data()); + shortSamples[0] = 12345; + shortSamples[1] = -12345; + std::vector shortOut; + convertAudioWithGain( + shortPkt.data(), + static_cast(shortPkt.size()), + source96k, + target48k, + 1.0, + shortOut); + const bool shortOk = !shortOut.empty() && target48k.blockAlign != 0 && + (shortOut.size() % target48k.blockAlign == 0); + expect( + "resample-96k-short-packet", + shortOk, + "bytes=" + std::to_string(shortOut.size())); + HRESULT mfHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); if (FAILED(mfHr) && mfHr != RPC_E_CHANGED_MODE) { skip("mf-startup", "CoInitializeEx failed — no Media Foundation on this host"); From 8919208a6f58660f9b4ddb8f2c6894103018b0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E8=B6=85=E7=BA=A7?= =?UTF-8?q?=E5=96=9C=E6=AC=A2=E4=BD=A0=E7=9A=84=E8=BE=BE=E5=A6=AE=E5=A8=85?= <176143450+My-Denia@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:04:01 +0800 Subject: [PATCH 4/4] fix: carry AAC downsample remainder across packets --- .../wgc-capture/src/audio_sample_utils.cpp | 102 +++++++++++++----- .../wgc-capture/src/audio_sample_utils.h | 13 ++- .../src/audio_sample_utils_test.cpp | 75 +++++++++++-- 3 files changed, 153 insertions(+), 37 deletions(-) diff --git a/electron/native/wgc-capture/src/audio_sample_utils.cpp b/electron/native/wgc-capture/src/audio_sample_utils.cpp index ec6ecdbd7..96847ee7d 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -189,6 +190,19 @@ void convertAudioWithGain( const AudioInputFormat& targetFormat, double gain, std::vector& destination) { + std::vector discardedRemainder; + convertAudioWithGain( + source, byteCount, sourceFormat, targetFormat, gain, destination, discardedRemainder); +} + +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination, + std::vector& remainder) { if (!source || byteCount == 0 || sourceFormat.blockAlign == 0 || targetFormat.blockAlign == 0 || sourceFormat.sampleRate == 0 || targetFormat.sampleRate == 0 || sourceFormat.channels == 0 || targetFormat.channels == 0) { @@ -201,8 +215,8 @@ void convertAudioWithGain( return; } - const size_t sourceFrames = byteCount / sourceFormat.blockAlign; - if (sourceFrames == 0) { + const size_t packetFrames = byteCount / sourceFormat.blockAlign; + if (packetFrames == 0) { destination.clear(); return; } @@ -210,37 +224,56 @@ void convertAudioWithGain( // Integer-factor downsample (96 kHz / 192 kHz -> 48 kHz): average each // group of source frames instead of picking one. Nearest-neighbour // decimation aliases content above the new Nyquist into the recording. + // Incomplete groups stay in remainder so the next packet can finish them. if (sourceFormat.sampleRate > targetFormat.sampleRate && sourceFormat.sampleRate % targetFormat.sampleRate == 0) { const UINT32 factor = sourceFormat.sampleRate / targetFormat.sampleRate; - const size_t targetFrames = sourceFrames / factor; - if (targetFrames > 0) { - destination.assign(targetFrames * targetFormat.blockAlign, 0); - for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { - for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { - double sum = 0.0; - for (UINT32 tap = 0; tap < factor; ++tap) { - sum += readMappedChannel( - source, - sourceFormat, - targetFrame * factor + tap, - channel, - targetFormat.channels); - } - writeSampleFromDouble( - destination.data(), - targetFormat, - targetFrame, + if (remainder.size() % sourceFormat.blockAlign != 0) { + remainder.clear(); + } + std::vector combined; + combined.reserve(remainder.size() + byteCount); + combined.insert(combined.end(), remainder.begin(), remainder.end()); + combined.insert(combined.end(), source, source + byteCount); + const size_t totalFrames = combined.size() / sourceFormat.blockAlign; + const size_t targetFrames = totalFrames / factor; + const size_t consumedFrames = targetFrames * factor; + const size_t leftoverBytes = (totalFrames - consumedFrames) * sourceFormat.blockAlign; + if (targetFrames == 0) { + destination.clear(); + remainder.swap(combined); + return; + } + destination.assign(targetFrames * targetFormat.blockAlign, 0); + for (size_t targetFrame = 0; targetFrame < targetFrames; ++targetFrame) { + for (UINT32 channel = 0; channel < targetFormat.channels; ++channel) { + double sum = 0.0; + for (UINT32 tap = 0; tap < factor; ++tap) { + sum += readMappedChannel( + combined.data(), + sourceFormat, + targetFrame * factor + tap, channel, - (sum / static_cast(factor)) * gain); + targetFormat.channels); } + writeSampleFromDouble( + destination.data(), + targetFormat, + targetFrame, + channel, + (sum / static_cast(factor)) * gain); } - return; } - // Too few source frames for one averaged output frame (tiny WASAPI - // packets). Fall through to interpolation instead of dropping them. + remainder.assign( + combined.begin() + static_cast(consumedFrames * sourceFormat.blockAlign), + combined.end()); + if (remainder.size() != leftoverBytes) { + remainder.resize(leftoverBytes); + } + return; } + const size_t sourceFrames = packetFrames; const double rateRatio = static_cast(targetFormat.sampleRate) / static_cast(sourceFormat.sampleRate); const size_t targetFrames = std::max(1, static_cast(std::llround(sourceFrames * rateRatio))); @@ -338,6 +371,8 @@ bool AudioMixer::start() { emittedFrames_ = 0; timelineStarted_ = false; paused_ = false; + systemResampleRemainder_.clear(); + microphoneResampleRemainder_.clear(); thread_ = std::thread([this] { mixLoop(); }); @@ -349,6 +384,8 @@ void AudioMixer::beginTimeline() { std::scoped_lock lock(mutex_); systemQueue_.clear(); microphoneQueue_.clear(); + systemResampleRemainder_.clear(); + microphoneResampleRemainder_.clear(); emittedFrames_ = 0; timelineStarted_ = true; } @@ -362,6 +399,8 @@ void AudioMixer::setPaused(bool paused) { if (paused_) { systemQueue_.clear(); microphoneQueue_.clear(); + systemResampleRemainder_.clear(); + microphoneResampleRemainder_.clear(); } } cv_.notify_all(); @@ -385,7 +424,7 @@ void AudioMixer::pushSystem(const BYTE* data, DWORD byteCount) { if (paused_) { return; } - append(systemQueue_, data, byteCount, systemFormat_, 1.0); + append(systemQueue_, data, byteCount, systemFormat_, 1.0, systemResampleRemainder_); } cv_.notify_all(); } @@ -400,7 +439,13 @@ void AudioMixer::pushMicrophone(const BYTE* data, DWORD byteCount) { if (paused_) { return; } - append(microphoneQueue_, data, byteCount, microphoneFormat_, microphoneGain_); + append( + microphoneQueue_, + data, + byteCount, + microphoneFormat_, + microphoneGain_, + microphoneResampleRemainder_); } cv_.notify_all(); } @@ -410,12 +455,13 @@ void AudioMixer::append( const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, - double gain) { + double gain, + std::vector& remainder) { if (!data || byteCount == 0) { return; } - convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_); + convertAudioWithGain(data, byteCount, sourceFormat, format_, gain, gainBuffer_, remainder); queue.insert(queue.end(), gainBuffer_.begin(), gainBuffer_.end()); } diff --git a/electron/native/wgc-capture/src/audio_sample_utils.h b/electron/native/wgc-capture/src/audio_sample_utils.h index 0bdbc0809..0f8e6b69f 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils.h +++ b/electron/native/wgc-capture/src/audio_sample_utils.h @@ -27,6 +27,14 @@ void convertAudioWithGain( const AudioInputFormat& targetFormat, double gain, std::vector& destination); +void convertAudioWithGain( + const BYTE* source, + DWORD byteCount, + const AudioInputFormat& sourceFormat, + const AudioInputFormat& targetFormat, + double gain, + std::vector& destination, + std::vector& remainder); void mixAudioInPlace( std::vector& destination, const BYTE* source, @@ -63,7 +71,8 @@ class AudioMixer { const BYTE* data, DWORD byteCount, const AudioInputFormat& sourceFormat, - double gain); + double gain, + std::vector& remainder); bool pop(std::vector& queue, std::vector& chunk, size_t byteCount); void mixLoop(); @@ -78,6 +87,8 @@ class AudioMixer { std::condition_variable cv_; std::vector systemQueue_; std::vector microphoneQueue_; + std::vector systemResampleRemainder_; + std::vector microphoneResampleRemainder_; std::vector gainBuffer_; std::thread thread_; std::atomic stopRequested_ = false; diff --git a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp index b261a7d28..8b74b9144 100644 --- a/electron/native/wgc-capture/src/audio_sample_utils_test.cpp +++ b/electron/native/wgc-capture/src/audio_sample_utils_test.cpp @@ -276,10 +276,21 @@ int main() { } expect("resample-96k-nyquist-box", folded, "frames=" + std::to_string(downFrames)); + auto fillStereoFrame = [](std::vector& packet, int16_t left, int16_t right) { + auto* samples = reinterpret_cast(packet.data()); + samples[0] = left; + samples[1] = right; + }; + const auto destFrames = [&](const std::vector& out) -> size_t { + return target48k.blockAlign == 0 ? 0 : out.size() / target48k.blockAlign; + }; + const auto remainderFrames = [&](const std::vector& rem) -> size_t { + return source96k.blockAlign == 0 ? 0 : rem.size() / source96k.blockAlign; + }; + + std::vector remainder; std::vector shortPkt(source96k.blockAlign, 0); - auto* shortSamples = reinterpret_cast(shortPkt.data()); - shortSamples[0] = 12345; - shortSamples[1] = -12345; + fillStereoFrame(shortPkt, 12345, -12345); std::vector shortOut; convertAudioWithGain( shortPkt.data(), @@ -287,13 +298,61 @@ int main() { source96k, target48k, 1.0, - shortOut); - const bool shortOk = !shortOut.empty() && target48k.blockAlign != 0 && - (shortOut.size() % target48k.blockAlign == 0); + shortOut, + remainder); expect( "resample-96k-short-packet", - shortOk, - "bytes=" + std::to_string(shortOut.size())); + shortOut.empty() && remainderFrames(remainder) == 1, + "dest=" + std::to_string(shortOut.size()) + " rem=" + std::to_string(remainder.size())); + + remainder.clear(); + std::vector oneA(source96k.blockAlign, 0); + std::vector oneB(source96k.blockAlign, 0); + fillStereoFrame(oneA, 1000, 2000); + fillStereoFrame(oneB, 3000, 4000); + std::vector outA; + std::vector outB; + convertAudioWithGain(oneA.data(), static_cast(oneA.size()), source96k, target48k, 1.0, outA, remainder); + convertAudioWithGain(oneB.data(), static_cast(oneB.size()), source96k, target48k, 1.0, outB, remainder); + expect( + "resample-96k-one-frame-packets", + destFrames(outA) == 0 && destFrames(outB) == 1 && remainder.empty(), + "a=" + std::to_string(destFrames(outA)) + " b=" + std::to_string(destFrames(outB)) + + " rem=" + std::to_string(remainderFrames(remainder))); + + remainder.clear(); + std::vector threePkt(3 * source96k.blockAlign, 0); + std::vector onePkt(source96k.blockAlign, 0); + fillStereoFrame(onePkt, 5000, 6000); + std::vector threeOut; + std::vector oneOut; + convertAudioWithGain( + threePkt.data(), static_cast(threePkt.size()), source96k, target48k, 1.0, threeOut, remainder); + convertAudioWithGain( + onePkt.data(), static_cast(onePkt.size()), source96k, target48k, 1.0, oneOut, remainder); + expect( + "resample-96k-remainder-three-then-one", + destFrames(threeOut) == 1 && destFrames(oneOut) == 1 && remainder.empty(), + "three=" + std::to_string(destFrames(threeOut)) + " one=" + std::to_string(destFrames(oneOut)) + + " rem=" + std::to_string(remainderFrames(remainder))); + + remainder.clear(); + std::vector remainderFullOut; + convertAudioWithGain( + source.data(), + static_cast(source.size()), + source96k, + target48k, + 1.0, + remainderFullOut, + remainder); + const size_t remainderFullFrames = destFrames(remainderFullOut); + const bool remainderFullOk = + remainderFullFrames == 48000 || remainderFullFrames == 47999 || remainderFullFrames == 48001; + expect( + "resample-96k-remainder-full", + remainderFullOk && remainder.empty(), + "frames=" + std::to_string(remainderFullFrames) + " rem=" + std::to_string(remainder.size())); HRESULT mfHr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); if (FAILED(mfHr) && mfHr != RPC_E_CHANGED_MODE) {