diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index de4defaad57cd..8a2b72d5b5df1 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -1345,7 +1345,6 @@ ur_result_t ur_command_list_manager::beginGraphCapture() { ZE2UR_CALL(hContextInternal->getPlatform() ->ZeGraphExt.zeCommandListBeginGraphCaptureExp, (getZeCommandList(), nullptr)); - graphCapture.enableCapture(); return UR_RESULT_SUCCESS; } @@ -1360,7 +1359,6 @@ ur_command_list_manager::beginCaptureIntoGraph(ur_exp_graph_handle_t hGraph) { ZE2UR_CALL(hContextInternal->getPlatform() ->ZeGraphExt.zeCommandListBeginCaptureIntoGraphExp, (getZeCommandList(), hGraph->getZeHandle(), nullptr)); - graphCapture.enableCapture(hGraph); return UR_RESULT_SUCCESS; } @@ -1375,18 +1373,17 @@ ur_command_list_manager::endGraphCapture(ur_exp_graph_handle_t *phGraph) { ze_graph_handle_t zeGraph = nullptr; ZE2UR_CALL(hContext.get()->getPlatform()->ZeGraphExt.endGraphCapture, (getZeCommandList(), nullptr, &zeGraph)); - auto graph = graphCapture.getGraph(); - graphCapture.disableCapture(); - if (!graph) { + { std::scoped_lock lock(hContextInternal->GraphMapMutex); - graph = hContextInternal->getGraphFromZeHandle(zeGraph); - if (!graph) { - graph = new ur_exp_graph_handle_t_(hContextInternal, zeGraph); - hContextInternal->registerGraph(zeGraph, graph); + auto hUrGraph = hContextInternal->getGraphFromZeHandle(zeGraph); + if (!hUrGraph) { + hUrGraph = new ur_exp_graph_handle_t_(hContextInternal, zeGraph); + hContextInternal->registerGraph(zeGraph, hUrGraph); } + + *phGraph = hUrGraph; } - *phGraph = graph; return UR_RESULT_SUCCESS; } @@ -1432,35 +1429,24 @@ ur_result_t ur_command_list_manager::getGraph(ur_exp_graph_handle_t *phGraph) { return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } - auto hCachedGraph = graphCapture.getGraph(); - if (hCachedGraph) { - *phGraph = hCachedGraph; - return UR_RESULT_SUCCESS; - } - - // Fork-join and implicit capture scenarios ze_graph_handle_t hZeGraph = nullptr; ze_result_t ZeResult = ZE_CALL_NOCHECK(zeGetGraph, (getZeCommandList(), &hZeGraph)); - if (ZeResult != ZE_RESULT_SUCCESS || !hZeGraph) { *phGraph = nullptr; return UR_RESULT_ERROR_INVALID_OPERATION; } - ur_exp_graph_handle_t hUrGraph = nullptr; { std::scoped_lock lock(hContextInternal->GraphMapMutex); - hUrGraph = hContextInternal->getGraphFromZeHandle(hZeGraph); + auto hUrGraph = hContextInternal->getGraphFromZeHandle(hZeGraph); if (!hUrGraph) { hUrGraph = new ur_exp_graph_handle_t_(hContextInternal, hZeGraph); hContextInternal->registerGraph(hZeGraph, hUrGraph); - if (graphCapture.isActive()) { - graphCapture.enableCapture(hUrGraph); - } } + + *phGraph = hUrGraph; } - *phGraph = hUrGraph; return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp index 8ba0bd797256e..35cd8e16a39ea 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.hpp @@ -46,25 +46,6 @@ struct wait_list_view { } }; -struct graph_capture_tracking_data_t { - void enableCapture(ur_exp_graph_handle_t graph) { - capturedGraph = graph; - enableCapture(); - } - void enableCapture() { active = true; } - void disableCapture() { - active = false; - capturedGraph = nullptr; - } - - ur_exp_graph_handle_t getGraph() const { return capturedGraph; } - bool isActive() const { return active; } - -private: - bool active = false; - ur_exp_graph_handle_t capturedGraph = nullptr; -}; - struct ur_command_list_manager { ur_command_list_manager(ur_context_handle_t hContext, ur_device_handle_t hDevice, @@ -336,8 +317,6 @@ struct ur_command_list_manager { std::unordered_set submittedKernels; v2::raii::command_list_unique_handle zeCommandList; std::vector waitList; - - graph_capture_tracking_data_t graphCapture; }; } // namespace ur::level_zero::v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp index a63232829f79b..7659856a0af1a 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp @@ -171,6 +171,14 @@ ur_result_t ur_queue_immediate_out_of_order_t::enqueueEventsWaitWithBarrierExt( : &ur_command_list_manager::appendEventsWait; auto commandListManagersLocked = commandListManagers.lock(); + bool captureActive = false; + commandListManagersLocked[captureCmdListManagerIdx].queryGraphCaptureActive( + &captureActive); + if (captureActive) { + return std::invoke( + barrierFn, commandListManagersLocked[captureCmdListManagerIdx], + waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); + } // Enqueue wait for the user-provider events on the first command list. UR_CALL(commandListManagersLocked[0].appendEventsWait(waitListView, diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp index 525a3b6802523..7f0beee35d58e 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp @@ -21,6 +21,8 @@ #include "lockable.hpp" #include "ur/ur.hpp" +#include + namespace ur::level_zero::v2 { struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { @@ -48,16 +50,39 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { std::array barrierEvents; - uint32_t getNextCommandListId() { + uint32_t getNextCommandListId(const ur_event_handle_t *phWaitEvents = nullptr, + uint32_t numWaitEvents = 0) { bool captureActive; auto &cmdListManager = (*commandListManagers.get_no_lock())[captureCmdListManagerIdx]; cmdListManager.queryGraphCaptureActive(&captureActive); - - return captureActive - ? captureCmdListManagerIdx - : commandListIndex.fetch_add(1, std::memory_order_relaxed) % - numCommandLists; + if (captureActive) { + return captureCmdListManagerIdx; + } + + // Any waitlist event that was produced by another queue that is + // currently recording a graph, this operation joins that capture and must + // be appended on the dedicated capture command list. Remember the + // originating ("primary") queue so that subsequent operations - even those + // without an explicit dependency on the primary queue - keep being routed + // onto the capture command list. The L0 driver then automatically enters + // capture mode on that command list, putting it into a temporary recording + // state that lasts until the primary queue stops recording. + for (uint32_t i = 0; i < numWaitEvents; i++) { + auto *srcQueue = phWaitEvents[i]->getQueue(); + if (!srcQueue || srcQueue == this) { + continue; + } + bool srcCaptureActive = false; + if (srcQueue->queueIsGraphCapteEnabledExp(&srcCaptureActive) == + UR_RESULT_SUCCESS && + srcCaptureActive) { + return captureCmdListManagerIdx; + } + } + + return commandListIndex.fetch_add(1, std::memory_order_relaxed) % + numCommandLists; } public: @@ -87,7 +112,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendEventsWait( waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); } @@ -106,7 +132,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferRead( hBuffer, blockingRead, offset, size, pDst, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -121,7 +148,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferWrite( hBuffer, blockingWrite, offset, size, pSrc, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -137,7 +165,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferReadRect( hBuffer, blockingRead, bufferOrigin, hostOrigin, region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, waitListView, @@ -154,7 +183,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferWriteRect( hBuffer, blockingWrite, bufferOrigin, hostOrigin, region, bufferRowPitch, bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, @@ -170,7 +200,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferCopy( hBufferSrc, hBufferDst, srcOffset, dstOffset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -186,7 +217,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferCopyRect( hBufferSrc, hBufferDst, srcOrigin, dstOrigin, region, srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, waitListView, @@ -202,7 +234,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferFill( hBuffer, pPattern, patternSize, offset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -218,7 +251,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageRead( hImage, blockingRead, origin, region, rowPitch, slicePitch, pDst, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -234,7 +268,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageWrite( hImage, blockingWrite, origin, region, rowPitch, slicePitch, pSrc, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -249,7 +284,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemImageCopy( hImageSrc, hImageDst, srcOrigin, dstOrigin, region, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -264,7 +300,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemBufferMap( hBuffer, blockingMap, mapFlags, offset, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this), ppRetMap); @@ -277,7 +314,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendMemUnmap( hMem, pMappedPtr, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -291,7 +329,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFill( pMem, patternSize, pPattern, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -304,7 +343,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMMemcpy( blocking, pDst, pSrc, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -318,7 +358,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFill2D( pMem, pitch, patternSize, pPattern, width, height, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -333,7 +374,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMMemcpy2D( blocking, pDst, dstPitch, pSrc, srcPitch, width, height, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -347,7 +389,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMPrefetch( pMem, size, flags, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -372,7 +415,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendDeviceGlobalVariableWrite( hProgram, name, blockingWrite, count, offset, pSrc, waitListView, @@ -387,7 +431,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendDeviceGlobalVariableRead( hProgram, name, blockingRead, count, offset, pDst, waitListView, @@ -403,7 +448,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendReadHostPipe( hProgram, pipe_symbol, blocking, pDst, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -418,7 +464,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendWriteHostPipe( hProgram, pipe_symbol, blocking, pSrc, size, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -432,7 +479,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -447,7 +495,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -463,7 +512,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMAllocHelper( this, pPool, size, pProperties, waitListView, ppMem, createEventIfRequested(eventPool.get(), phEvent, this), @@ -477,7 +527,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendUSMFreeExp( this, pPool, pMem, waitListView, createEvent(eventPool.get(), phEvent, this)); @@ -496,7 +547,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].bindlessImagesImageCopyExp( pSrc, pDst, pSrcImageDesc, pDstImageDesc, pSrcImageFormat, pDstImageFormat, pCopyRegion, imageCopyFlags, imageCopyInputTypes, @@ -511,7 +563,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .bindlessImagesWaitExternalSemaphoreExp( hSemaphore, hasWaitValue, waitValue, waitListView, @@ -526,7 +579,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .bindlessImagesSignalExternalSemaphoreExp( hSemaphore, hasSignalValue, signalValue, waitListView, @@ -540,7 +594,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendTimestampRecordingExp( blocking, waitListView, @@ -555,7 +610,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendCommandBufferExp( hCommandBuffer, waitListView, createEventAndRetain(eventPool.get(), phEvent, this)); @@ -570,7 +626,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendNativeCommandExp( pfnNativeEnqueue, data, numMemsInMemList, phMemList, pProperties, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); @@ -587,7 +644,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId] .appendKernelLaunchWithArgsExp( hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, @@ -618,15 +676,18 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendGraph( hGraph, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); } ur_result_t queueIsGraphCapteEnabledExp(bool *pResult) override { - return commandListManagers.lock()[captureCmdListManagerIdx] - .queryGraphCaptureActive(pResult); + UR_CALL(commandListManagers.lock()[captureCmdListManagerIdx] + .queryGraphCaptureActive(pResult)); + + return UR_RESULT_SUCCESS; } ur_result_t queueGetGraphExp(ur_exp_graph_handle_t *phGraph) override { @@ -643,7 +704,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object_t, ur_queue_t_ { wait_list_view waitListView = wait_list_view(phEventWaitList, numEventsInWaitList); - auto commandListId = getNextCommandListId(); + auto commandListId = + getNextCommandListId(phEventWaitList, numEventsInWaitList); return commandListManagers.lock()[commandListId].appendHostTaskExp( pfnHostTask, data, pProperties, waitListView, createEventIfRequested(eventPool.get(), phEvent, this)); diff --git a/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp index 1d9cbb72bd38a..a65fba13ac4e1 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueGetGraphExp.cpp @@ -96,11 +96,6 @@ struct urQueueGetGraphExpMultiQueueTest // Fork-join was initially broken with zeCommandListGetGraph std::tuple minL0DriverVersion = {1, 15, 38146}; SKIP_IF_DRIVER_TOO_OLD("Level-Zero", minL0DriverVersion, platform, device); - - // Fork-join with out-of-order queue broken due to multi command list capture bug - if (getQueueFlag() & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - } } void TearDown() override { bool isCaptureEnabled = false; diff --git a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp index 95838078737ce..858b13eca81c1 100644 --- a/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp +++ b/unified-runtime/test/conformance/exp_graph/urQueueIsGraphCaptureEnabledExp.cpp @@ -4,6 +4,10 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include "fixtures.h" +#include "unified-runtime/ur_api.h" +#include "uur/raii.h" + +#include struct urQueueIsGraphCaptureEnabledExpTest : uur::urGraphSupportedExpTest { void SetUp() override { @@ -67,10 +71,6 @@ struct urQueueIsGraphCaptureEnabledExpMultiQueueTest : uur::urGraphSupportedExpMultiQueueTest { void SetUp() override { UUR_RETURN_ON_FATAL_FAILURE(urGraphSupportedExpMultiQueueTest::SetUp()); - // Fork-join with out-of-order queue broken due to multi command list capture bug - if (getQueueFlag() & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) { - UUR_KNOWN_FAILURE_ON(uur::LevelZeroV2{}); - } } void TearDown() override { bool isCaptureEnabled = false; @@ -133,3 +133,155 @@ TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, ForkJoinPattern) { ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); ASSERT_FALSE(isEnabled); } + +// Tests that, once an out-of-order queue has joined a capture via the fork-join +// pattern, operations subsequently submitted to it *without* an explicit +// dependency on the recording queue are still recorded. The secondary queue +// stays in the temporary recording state until the primary queue ends the +// capture. With the L0v2 out-of-order queue this would otherwise round-robin +// the dependency-less operation onto a non-capture command list, escaping the +// capture. +TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, + ForkJoinSubsequentOpsWithoutDependency) { + bool isEnabled = false; + + // Advance the out-of-order queue's command list selection so the next + // operation would not land on the dedicated capture command list by chance. + uur::raii::Event preEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue2, 0, nullptr, preEvent.ptr())); + ASSERT_SUCCESS(urEventWait(1, preEvent.ptr())); + + ASSERT_SUCCESS(urQueueBeginGraphCaptureExp(queue1)); + + // Fork: queue1 produces an event that queue2 waits on, pulling queue2 into + // the capture. + uur::raii::Event forkEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 0, nullptr, forkEvent.ptr())); + + uur::raii::Event joinEvent = nullptr; + ASSERT_SUCCESS( + urEnqueueEventsWait(queue2, 1, forkEvent.ptr(), joinEvent.ptr())); + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_TRUE(isEnabled); + + // Subsequent operation on queue2 without any dependency on queue1. It must + // remain part of the capture: queue2 should still report recording enabled. + uur::raii::Event noDepEvent = nullptr; + size_t size = 1024; + void *ptr1 = nullptr; + void *ptr2 = nullptr; + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr1)); + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr2)); + + // Fill ptr1 with a pattern and clear ptr2 so we can tell whether the copy + // was recorded or executed. + uint32_t *ptr1_data = static_cast(ptr1); + uint32_t *ptr2_data = static_cast(ptr2); + *ptr1_data = 0xdeadbeefU; + *ptr2_data = 0U; + + // Submit memcpy operation on queue2 without dependency on queue1. This + // operation should be recorded in the capture. It must be non-blocking: a + // blocking copy would host-synchronize the capturing command list, which + // has nothing to wait for because the copy is only recorded, and the driver + // rejects the synchronization. + ASSERT_SUCCESS(urEnqueueUSMMemcpy(queue2, false, ptr2, ptr1, size, 0, nullptr, + noDepEvent.ptr())); + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_TRUE(isEnabled); + + // Verify that the copy is not performed yet (it's recorded but not executed) + ASSERT_EQ(*ptr2_data, 0U); + + // Join both queue2 operations back to queue1 and finish recording. + ur_event_handle_t joinEvents[] = {joinEvent.get(), noDepEvent.get()}; + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 2, joinEvents, nullptr)); + + ASSERT_SUCCESS(urQueueEndGraphCaptureExp(queue1, &graph)); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue1, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // The primary queue stopped recording, so queue2 must leave the temporary + // recording state as well. + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // Replaying the captured graph must execute the copy that was recorded on + // the forked queue, proving it really became part of the graph. + ur_exp_executable_graph_handle_t exGraph = nullptr; + ASSERT_SUCCESS(urGraphInstantiateGraphExp(graph, &exGraph)); + + ASSERT_SUCCESS(urEnqueueGraphExp(queue1, exGraph, 0, nullptr, nullptr)); + ASSERT_SUCCESS(urQueueFinish(queue1)); + + EXPECT_EQ(*ptr2_data, 0xdeadbeefU); + + EXPECT_SUCCESS(urGraphExecutableGraphDestroyExp(exGraph)); + ASSERT_SUCCESS(urUSMFree(context, ptr1)); + ASSERT_SUCCESS(urUSMFree(context, ptr2)); +} + +TEST_P(urQueueIsGraphCaptureEnabledExpMultiQueueTest, ForkJoinBackToState) { + bool isEnabled = false; + + // Advance the out-of-order queue's command list selection so the next + // operation would not land on the dedicated capture command list by chance. + uur::raii::Event preEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue2, 0, nullptr, preEvent.ptr())); + ASSERT_SUCCESS(urEventWait(1, preEvent.ptr())); + + ASSERT_SUCCESS(urQueueBeginGraphCaptureExp(queue1)); + + // Fork: queue1 produces an event that queue2 waits on, pulling queue2 into + // the capture. + uur::raii::Event forkEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 0, nullptr, forkEvent.ptr())); + + uur::raii::Event joinEvent = nullptr; + ASSERT_SUCCESS( + urEnqueueEventsWait(queue2, 1, forkEvent.ptr(), joinEvent.ptr())); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue1, &isEnabled)); + ASSERT_TRUE(isEnabled); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_TRUE(isEnabled); + ASSERT_SUCCESS(urEnqueueEventsWait(queue1, 1, joinEvent.ptr(), nullptr)); + + ASSERT_SUCCESS(urQueueEndGraphCaptureExp(queue1, &graph)); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue1, &isEnabled)); + ASSERT_FALSE(isEnabled); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // Starts capture again on queue1, which should not affect queue2. Queue2 should remain in the default non-capturing state. + ASSERT_SUCCESS(urQueueBeginGraphCaptureExp(queue1)); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue1, &isEnabled)); + ASSERT_TRUE(isEnabled); + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // Enqueue new operation that should be executed immediately on queue2, which is not capturing. This should not trigger any capture state change. + uur::raii::Event newEvent = nullptr; + ASSERT_SUCCESS(urEnqueueEventsWait(queue2, 0, nullptr, newEvent.ptr())); + size_t size = 1024; + void *ptr1 = nullptr; + char pattern[8] = {0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8}; + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, size, &ptr1)); + std::memset(ptr1, 0, size); + ASSERT_SUCCESS(urEnqueueUSMFill(queue2, ptr1, std::size(pattern), pattern, + size, 0, nullptr, newEvent.ptr())); + + ASSERT_SUCCESS(urQueueIsGraphCaptureEnabledExp(queue2, &isEnabled)); + ASSERT_FALSE(isEnabled); + + // Not captured, so the fill must have executed immediately. + ASSERT_SUCCESS(urQueueFinish(queue2)); + EXPECT_EQ(std::memcmp(ptr1, pattern, std::size(pattern)), 0); + + ASSERT_SUCCESS(urUSMFree(context, ptr1)); +} \ No newline at end of file