diff --git a/test/query-test-suite/CMakeLists.txt b/test/query-test-suite/CMakeLists.txt index 4592704b0c..095c237e18 100644 --- a/test/query-test-suite/CMakeLists.txt +++ b/test/query-test-suite/CMakeLists.txt @@ -17,6 +17,16 @@ function(add_query_suite_target exe_name) turing_testutils_s) endfunction() +set(V3_QUERY_SUITE_LIBS + turing_db_interpreter_v3_s + turing_db_ir_codegen_s + turing_db_ir_dbdialect_s + turing_db_ir_nldialect_s + turing_db_ir_storagedialect_s + turing_db_ir_interpreter_s + turing_db_ir_common_s + turing_db_frontend_cypher_s) + add_query_suite_target(test_query_test_suite QueryTestSuite.cpp QueryTestRunner.cpp @@ -40,15 +50,30 @@ gtest_discover_tests(test_remote_query_test_suite DISCOVERY_TIMEOUT 10 DISCOVER_MODE PRE_TEST) +add_query_suite_target(test_query_v3_test_suite + V3QueryTestSuite.cpp + V3QueryTestRunner.cpp + QueryTestRunner.cpp + QueryResultFormatter.cpp) + +target_link_libraries(test_query_v3_test_suite + PRIVATE ${V3_QUERY_SUITE_LIBS}) + +gtest_discover_tests(test_query_v3_test_suite + DISCOVERY_TIMEOUT 10 + DISCOVER_MODE PRE_TEST) + add_query_suite_target(query_test_suite_cli QueryTestSuiteCLI.cpp QueryTestRunner.cpp RemoteQueryTestRunner.cpp + V3QueryTestRunner.cpp QueryResultFormatter.cpp) target_link_libraries(query_test_suite_cli PRIVATE turing_db_server_s - turing_db_proto_client_s) + turing_db_proto_client_s + ${V3_QUERY_SUITE_LIBS}) set(QUERY_TEST_SUITE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/tests) @@ -58,5 +83,8 @@ target_compile_definitions(test_query_test_suite target_compile_definitions(test_remote_query_test_suite PRIVATE QUERY_TEST_SUITE_DIR="${QUERY_TEST_SUITE_DIR}") +target_compile_definitions(test_query_v3_test_suite + PRIVATE QUERY_TEST_SUITE_DIR="${QUERY_TEST_SUITE_DIR}") + target_compile_definitions(query_test_suite_cli PRIVATE QUERY_TEST_SUITE_DIR="${QUERY_TEST_SUITE_DIR}") diff --git a/test/query-test-suite/QueryResultFormatter.cpp b/test/query-test-suite/QueryResultFormatter.cpp index 610ffe00a6..66be7cdcc8 100644 --- a/test/query-test-suite/QueryResultFormatter.cpp +++ b/test/query-test-suite/QueryResultFormatter.cpp @@ -272,6 +272,23 @@ void QueryResultFormatter::appendRows(std::vector>& row } } +void QueryResultFormatter::appendChunkRows(std::vector>& rows, + std::vector& values, + std::span chunks, + size_t offset, + size_t rowCount) { + for (size_t row = offset; row < offset + rowCount; ++row) { + values.clear(); + values.reserve(chunks.size()); + + for (const db::Column* col : chunks) { + values.push_back(columnValueToString(col, row)); + } + + rows.push_back(values); + } +} + std::string QueryResultFormatter::formatResultOutput(const db::QueryStatus& status, const std::vector& columnNames, const std::vector>& rows) { diff --git a/test/query-test-suite/QueryResultFormatter.h b/test/query-test-suite/QueryResultFormatter.h index a140f4e574..b45d176334 100644 --- a/test/query-test-suite/QueryResultFormatter.h +++ b/test/query-test-suite/QueryResultFormatter.h @@ -1,10 +1,12 @@ #pragma once +#include #include #include namespace db { +class Column; class Dataframe; class QueryStatus; @@ -21,6 +23,12 @@ class QueryResultFormatter { std::vector& values, const db::Dataframe* df); + static void appendChunkRows(std::vector>& rows, + std::vector& values, + std::span chunks, + size_t offset, + size_t rowCount); + static std::string formatResultOutput(const db::QueryStatus& status, const std::vector& columnNames, const std::vector>& rows); diff --git a/test/query-test-suite/QueryTestRunner.cpp b/test/query-test-suite/QueryTestRunner.cpp index 3266de582a..d8ce2d5d01 100644 --- a/test/query-test-suite/QueryTestRunner.cpp +++ b/test/query-test-suite/QueryTestRunner.cpp @@ -301,6 +301,7 @@ void QueryTestRunner::loadTestsFromDir(std::vector& specs, spec._expectPlan = expect.value("plan", ""); spec._expectResult = expect.value("result", ""); spec._expectResultJson = expect.value("resultJson", ""); + spec._expectMlir = expect.value("mlir", ""); } specs.push_back(std::move(spec)); diff --git a/test/query-test-suite/QueryTestSuiteCLI.cpp b/test/query-test-suite/QueryTestSuiteCLI.cpp index 88c91fee59..9fd2b86a38 100644 --- a/test/query-test-suite/QueryTestSuiteCLI.cpp +++ b/test/query-test-suite/QueryTestSuiteCLI.cpp @@ -6,6 +6,7 @@ #include "QueryTestRunner.h" #include "RemoteQueryTestRunner.h" +#include "V3QueryTestRunner.h" using namespace turing::test; @@ -110,6 +111,18 @@ std::string serializeResult(const QueryTestResult& result, bool includeJsonField result._resultJsonValid ? "true" : "false", result._timeUs); } +std::string serializeResultV3(const V3QueryTestResult& result) { + return fmt::format( + "{{\"name\":\"{}\",\"resultV3Output\":\"{}\"," + "\"mlirProgram\":\"{}\"," + "\"resultV3Matched\":{},\"mlirMatched\":{}," + "\"timeUs\":{}}}", + escapeJson(result._name), escapeJson(result._resultOutput), + escapeJson(result._mlirOutput), + result._resultMatched ? "true" : "false", + result._mlirMatched ? "true" : "false", result._timeUs); +} + } // namespace int main(int argc, char** argv) { @@ -128,6 +141,10 @@ int main(int argc, char** argv) { .help("Run a single test by name through the remote protocol") .metavar("name") .nargs(1); + program.add_argument("--run-v3") + .help("Run a single test by name through the v3 MLIR interpreter") + .metavar("name") + .nargs(1); program.add_argument("--run-all") .help("Run all enabled tests") .default_value(false) @@ -136,6 +153,10 @@ int main(int argc, char** argv) { .help("Run all enabled tests through the remote protocol") .default_value(false) .implicit_value(true); + program.add_argument("--run-all-v3") + .help("Run all enabled tests through the v3 MLIR interpreter") + .default_value(false) + .implicit_value(true); try { program.parse_args(argc, argv); @@ -148,10 +169,15 @@ int main(int argc, char** argv) { const bool doList = program.get("--list"); const bool doRunAll = program.get("--run-all"); const bool doRunAllRemote = program.get("--run-all-remote"); + const bool doRunAllV3 = program.get("--run-all-v3"); const bool doRun = program.is_used("--run"); const bool doRunRemote = program.is_used("--run-remote"); + const bool doRunV3 = program.is_used("--run-v3"); - if ((doList ? 1 : 0) + (doRun ? 1 : 0) + (doRunRemote ? 1 : 0) + (doRunAll ? 1 : 0) + (doRunAllRemote ? 1 : 0) != 1) { + const int selectedModes = (doList ? 1 : 0) + (doRun ? 1 : 0) + (doRunRemote ? 1 : 0) + + (doRunV3 ? 1 : 0) + (doRunAll ? 1 : 0) + (doRunAllRemote ? 1 : 0) + + (doRunAllV3 ? 1 : 0); + if (selectedModes != 1) { fmt::println("{}", argParserUsage(program)); return 1; } @@ -181,6 +207,7 @@ int main(int argc, char** argv) { QueryTestRunner runner; RemoteQueryTestRunner remoteRunner; + V3QueryTestRunner v3Runner; if (doRun) { const std::string name = program.get("--run"); @@ -218,6 +245,46 @@ int main(int argc, char** argv) { return 1; } + if (doRunV3) { + const std::string name = program.get("--run-v3"); + for (const auto& test : tests) { + if (test._name != name) { + continue; + } + const fs::Path outDir = fs::Path {"query_test_suite_cli_v3"} / test._name; + const V3QueryTestResult result = v3Runner.runTest(test, outDir); + fmt::println("{}", serializeResultV3(result)); + return 0; + } + fmt::println("{}", "{\"error\":\"Unknown test name\"}"); + return 1; + } + + if (doRunAllV3) { + fmt::print("["); + bool first = true; + for (const auto& test : tests) { + if (!test._enabled) { + continue; + } + V3QueryTestResult result; + try { + result = v3Runner.runTest(test, fs::Path {"query_test_suite_cli_v3"} / test._name); + } catch (const std::exception& e) { + result._name = test._name; + result._resultOutput = fmt::format("ERROR: {}", e.what()); + } + + if (!first) { + fmt::print(","); + } + first = false; + fmt::print("{}", serializeResultV3(result)); + } + fmt::println("]"); + return 0; + } + fmt::print("["); bool first = true; for (const auto& test : tests) { diff --git a/test/query-test-suite/QueryTestTypes.h b/test/query-test-suite/QueryTestTypes.h index 19bd0c4065..67c7491fc3 100644 --- a/test/query-test-suite/QueryTestTypes.h +++ b/test/query-test-suite/QueryTestTypes.h @@ -13,6 +13,7 @@ struct QueryTestSpec { std::string _expectPlan; std::string _expectResult; std::string _expectResultJson; + std::string _expectMlir; std::vector _tags; bool _enabled {true}; bool _remoteEnabled {true}; @@ -34,4 +35,13 @@ struct QueryTestResult { uint64_t _timeUs {0}; }; +struct V3QueryTestResult { + std::string _name; + std::string _resultOutput; + std::string _mlirOutput; + bool _resultMatched {false}; + bool _mlirMatched {false}; + uint64_t _timeUs {0}; +}; + } diff --git a/test/query-test-suite/V3QueryTestRunner.cpp b/test/query-test-suite/V3QueryTestRunner.cpp new file mode 100644 index 0000000000..f54733685c --- /dev/null +++ b/test/query-test-suite/V3QueryTestRunner.cpp @@ -0,0 +1,251 @@ +#include "V3QueryTestRunner.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "llvm/Support/raw_ostream.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/MLIRContext.h" +#include "mlir/IR/OwningOpRef.h" + +#include "DBDialect.h" +#include "DBProgramGenerator.h" +#include "NLDialect.h" +#include "NLOutputSink.h" +#include "StorageDialect.h" + +#include "BioAssert.h" +#include "CompilerException.h" +#include "CypherAST.h" +#include "CypherAnalyzer.h" +#include "CypherParser.h" +#include "Projection.h" +#include "QueryCommand.h" +#include "SinglePartQuery.h" +#include "TuringException.h" +#include "stmt/ReturnStmt.h" + +#include "Graph.h" +#include "ID.h" +#include "ProcedureManager.h" +#include "QueryCallbacks.h" +#include "QueryConfig.h" +#include "QueryInterpreterV3.h" +#include "QueryResultFormatter.h" +#include "QueryState.h" +#include "QueryStatus.h" +#include "QueryTestRunner.h" +#include "SimpleGraph.h" +#include "SystemAccessor.h" +#include "SystemManager.h" +#include "TuringDB.h" +#include "TuringTestEnv.h" +#include "TuringTime.h" +#include "columns/ColumnVector.h" +#include "dataframe/Dataframe.h" +#include "dataframe/NamedColumn.h" +#include "versioning/ChangeID.h" +#include "versioning/CommitHash.h" +#include "versioning/Transaction.h" +#include "views/GraphView.h" + +using namespace db; + +namespace turing::test { + +namespace { + +class CollectingNLSink : public NLOutputSink { +public: + explicit CollectingNLSink(std::vector>& rows) + : _rows(rows) + { + } + + void appendChunks(std::span chunks, size_t offset, size_t rowCount) override { + QueryResultFormatter::appendChunkRows(_rows, _values, chunks, offset, rowCount); + } + +private: + std::vector>& _rows; + std::vector _values; +}; + +// The v3 result must line up with the shared expect.result, so its header uses the +// same column names the v2 pipeline assigns - the analyzed RETURN projection, read in +// the same order as PipelineGenerator::translateProduceResultsNode. +void collectReturnColumnNames(const CypherAST& ast, std::vector& columnNames) { + columnNames.clear(); + + const Projection* projection = nullptr; + for (const QueryCommand* command : ast.queries()) { + if (command->getKind() != QueryCommand::Kind::SINGLE_PART_QUERY) { + continue; + } + + const SinglePartQuery* query = static_cast(command); + const ReturnStmt* returnStmt = query->getReturnStmt(); + if (returnStmt) { + projection = returnStmt->getProjection(); + } + } + + if (!projection) { + return; + } + + for (const Projection::ReturnItem& item : projection->items()) { + std::optional name; + if (Expr* const* exprPtr = std::get_if(&item)) { + name = projection->getName(*exprPtr); + } else if (VarDecl* const* declPtr = std::get_if(&item)) { + name = projection->getName(*declPtr); + } + + if (name) { + columnNames.emplace_back(*name); + } + } +} + +void generateMLIRProgram(std::string& out, + std::vector& columnNames, + std::string_view query, + GraphView view) { + out.clear(); + columnNames.clear(); + + auto procedures = std::make_unique(); + procedures->init(); + + CypherAST ast(procedures.get(), query); + CypherParser parser(&ast); + try { + parser.parse(query); + } catch (const CompilerException& e) { + out = fmt::format("PARSE ERROR\n{}", e.what()); + return; + } + + CypherAnalyzer analyzer(&ast, view); + analyzer.setV3(); + try { + analyzer.analyze(); + } catch (const CompilerException& e) { + out = fmt::format("ANALYZE ERROR\n{}", e.what()); + return; + } + + collectReturnColumnNames(ast, columnNames); + + mlir::MLIRContext context; + context.getOrLoadDialect(); + context.getOrLoadDialect(); + context.getOrLoadDialect(); + context.getOrLoadDialect(); + + mlir::OpBuilder builder(&context); + mlir::OwningOpRef owningModule = mlir::ModuleOp::create(builder.getUnknownLoc()); + mlir::ModuleOp module = owningModule.get(); + + DBProgramGenerator generator(&module); + try { + generator.generate(&ast); + } catch (const CompilerException& e) { + out = fmt::format("PLAN ERROR\n{}", e.what()); + return; + } catch (const TuringException& e) { + out = fmt::format("PLAN ERROR\n{}", e.what()); + return; + } + + llvm::raw_string_ostream stream(out); + module.print(stream); +} + +} + +V3QueryTestResult V3QueryTestRunner::runTest(const QueryTestSpec& spec, const fs::Path& outDir) { + V3QueryTestResult result; + result._name = spec._name; + + auto env = turing::test::TuringTestEnv::create(outDir); + Graph* graph = nullptr; + { + SystemAccessor system = env->getSystemManager().accessUnique(); + graph = system.createGraph(spec._graphName); + } + SimpleGraph::createSimpleGraph(graph); + TuringDB* db = &env->getDB(); + + std::string mlirOutput; + std::vector columnNames; + { + const Transaction tx = graph->openTransaction(); + const GraphView view = tx.viewGraph(); + generateMLIRProgram(mlirOutput, columnNames, spec._query, view); + } + + QueryConfig queryConfig; + + ChangeID changeID = ChangeID::head(); + if (spec._writeRequired) { + QueryCallbacks changeNewCallbacks; + changeNewCallbacks.setOnOutputData([&](const Dataframe* df) { + NamedColumn* col = df->getColumn(ColumnTag {0}); + bioassert(col, "Column not found"); + + ColumnVector& changeIDs = *static_cast*>(col->getColumn()); + bioassert(changeIDs.size() == 1, "Expected 1 change"); + + changeID = changeIDs[0]; + }); + + const QueryState changeNewState(spec._graphName, &env->getMem(), &queryConfig, &changeNewCallbacks); + db->query("CHANGE NEW", changeNewState); + } + + std::vector> rows; + CollectingNLSink sink(rows); + + QueryStatus status; + QueryInterpreterV3 interpreter(&env->getSystemManager()); + + const auto queryStart = Clock::now(); + interpreter.execute(status, spec._query, spec._graphName, CommitHash::head(), changeID, &env->getMem(), &sink); + const auto queryEnd = Clock::now(); + + result._timeUs = static_cast(duration(queryStart, queryEnd)); + + if (spec._writeRequired) { + QueryCallbacks submitCallbacks; + const QueryState submitState(spec._graphName, &env->getMem(), &queryConfig, &submitCallbacks, + CommitHash::head(), changeID); + db->query("CHANGE SUBMIT", submitState); + } + + QueryTestRunner::normalizeOutput( + result._resultOutput, + QueryResultFormatter::formatResultOutput(status, columnNames, rows)); + QueryTestRunner::normalizeOutput(result._mlirOutput, mlirOutput); + + std::string expected; + + QueryTestRunner::normalizeOutput(expected, spec._expectResult); + result._resultMatched = expected == result._resultOutput; + + QueryTestRunner::normalizeOutput(expected, spec._expectMlir); + result._mlirMatched = expected == result._mlirOutput; + + return result; +} + +} diff --git a/test/query-test-suite/V3QueryTestRunner.h b/test/query-test-suite/V3QueryTestRunner.h new file mode 100644 index 0000000000..395fc925e3 --- /dev/null +++ b/test/query-test-suite/V3QueryTestRunner.h @@ -0,0 +1,13 @@ +#pragma once + +#include "Path.h" +#include "QueryTestTypes.h" + +namespace turing::test { + +class V3QueryTestRunner { +public: + V3QueryTestResult runTest(const QueryTestSpec& spec, const fs::Path& outDir); +}; + +} diff --git a/test/query-test-suite/V3QueryTestSuite.cpp b/test/query-test-suite/V3QueryTestSuite.cpp new file mode 100644 index 0000000000..ba20035c17 --- /dev/null +++ b/test/query-test-suite/V3QueryTestSuite.cpp @@ -0,0 +1,49 @@ +#include "TuringTest.h" + +#include "QueryTestRunner.h" +#include "V3QueryTestRunner.h" + +namespace turing::test { + +class V3QueryTestSuite : public TuringTest { +public: + void initialize() override { + QueryTestRunner::loadTestsFromDir(_tests, fs::Path {QUERY_TEST_SUITE_DIR}); + } + +protected: + std::vector _tests; +}; + +TEST_F(V3QueryTestSuite, RunAll) { + V3QueryTestRunner runner; + size_t executed = 0; + std::string normalized; + + for (const auto& test : _tests) { + if (!test._enabled) { + continue; + } + + ++executed; + const fs::Path outDir = fs::Path {_outDir} / test._name; + const V3QueryTestResult result = runner.runTest(test, outDir); + + if (!result._resultMatched) { + QueryTestRunner::normalizeOutput(normalized, test._expectResult); + ADD_FAILURE() << "V3 result output mismatch for test: " << test._name; + ADD_FAILURE() << "Expected result:\n" + << normalized; + ADD_FAILURE() << "Actual result:\n" + << result._resultOutput; + } + } + + EXPECT_GE(executed, 0u); +} + +} + +int main(int argc, char** argv) { + return turing::test::turingTestMain(argc, argv, [] { testing::GTEST_FLAG(repeat) = 1; }); +} diff --git a/test/test-suite/backend/.gitignore b/test/test-suite/backend/.gitignore new file mode 100644 index 0000000000..66819bf10c --- /dev/null +++ b/test/test-suite/backend/.gitignore @@ -0,0 +1,4 @@ +# Scratch output the query test CLI writes under the server's working directory +query_test_suite_cli/ +query_test_suite_cli_remote/ +query_test_suite_cli_v3/ diff --git a/test/test-suite/backend/server-utils.ts b/test/test-suite/backend/server-utils.ts index 0e5855eb76..235687214f 100644 --- a/test/test-suite/backend/server-utils.ts +++ b/test/test-suite/backend/server-utils.ts @@ -16,12 +16,14 @@ export type ExpectedOutput = { plan: string; result: string; resultJson: string; + mlir: string; }; export type UpdateTestOptions = { plan?: string; result?: string; resultJson?: string; + mlir?: string; query?: string; newName?: string; tags?: string[]; @@ -173,6 +175,7 @@ export async function loadMainExpected( plan: typeof expect.plan === "string" ? expect.plan : "", result: typeof expect.result === "string" ? expect.result : "", resultJson: typeof expect.resultJson === "string" ? expect.resultJson : "", + mlir: typeof expect.mlir === "string" ? expect.mlir : "", }; } @@ -211,6 +214,7 @@ export async function loadExpectedFromFile( plan: typeof expect.plan === "string" ? expect.plan : "", result: typeof expect.result === "string" ? expect.result : "", resultJson: typeof expect.resultJson === "string" ? expect.resultJson : "", + mlir: typeof expect.mlir === "string" ? expect.mlir : "", }; } @@ -274,6 +278,9 @@ export async function updateTestFile( if (typeof options.resultJson === "string") { expect.resultJson = options.resultJson; } + if (typeof options.mlir === "string") { + expect.mlir = options.mlir; + } if (typeof options.query === "string") { data.query = options.query; } @@ -337,6 +344,7 @@ export async function createTestFile( plan: "", result: "", resultJson: "", + mlir: "", }, tags: [] as string[], "write-required": false, diff --git a/test/test-suite/backend/server.ts b/test/test-suite/backend/server.ts index 2db77b9147..562131f844 100644 --- a/test/test-suite/backend/server.ts +++ b/test/test-suite/backend/server.ts @@ -70,6 +70,10 @@ async function runCliJsonResponse( Bun.serve({ port: PORT, + // A full run-all invocation shells out to the C++ CLI and streams nothing back + // until every test has run, so the connection sits idle far longer than Bun's + // 10s default. 255 is the maximum idleTimeout Bun accepts. + idleTimeout: 255, async fetch(req) { const { pathname, searchParams } = new URL(req.url); @@ -181,6 +185,18 @@ Bun.serve({ ); } + if (pathname === "/api/run-v3") { + const testId = searchParams.get("test"); + if (!testId) { + return errorResponse("Missing test parameter", 400); + } + return runCliJsonResponse( + [ "--run-v3", testId ], + "Failed to run v3 test", + "Invalid v3 run response", + ); + } + if (pathname === "/api/run-all") { return runCliJsonResponse( ["--run-all"], @@ -197,11 +213,20 @@ Bun.serve({ ); } + if (pathname === "/api/run-all-v3") { + return runCliJsonResponse( + ["--run-all-v3"], + "Failed to run all v3 tests", + "Invalid v3 run-all response", + ); + } + if (pathname === "/api/update" && req.method === "POST") { const body = await req.json().catch(() => null); const hasPlan = typeof body?.plan === "string"; const hasResult = typeof body?.result === "string"; const hasResultJson = typeof body?.resultJson === "string"; + const hasMlir = typeof body?.mlir === "string"; const hasQuery = typeof body?.query === "string"; const hasNewName = typeof body?.newName === "string"; const hasTags = Array.isArray(body?.tags); @@ -215,6 +240,7 @@ Bun.serve({ (!hasPlan && !hasResult && !hasResultJson && + !hasMlir && !hasQuery && !hasNewName && !hasTags && @@ -238,6 +264,7 @@ Bun.serve({ plan: body.plan, result: body.result, resultJson: body.resultJson, + mlir: body.mlir, query: body.query, newName: body.newName, tags: body.tags, @@ -251,6 +278,7 @@ Bun.serve({ plan: body.plan, result: body.result, resultJson: body.resultJson, + mlir: body.mlir, query: body.query, newName: body.newName, tags: body.tags, diff --git a/test/test-suite/frontend/src/App.tsx b/test/test-suite/frontend/src/App.tsx index ba26e9aa9c..72847e7d23 100644 --- a/test/test-suite/frontend/src/App.tsx +++ b/test/test-suite/frontend/src/App.tsx @@ -41,7 +41,7 @@ type TestMeta = { disabledReason?: string; mainVersion?: { query?: string; - expect?: { plan?: string; result?: string; resultJson?: string }; + expect?: { plan?: string; result?: string; resultJson?: string; mlir?: string }; tags?: string[]; enabled?: boolean; ["write-required"]?: boolean; @@ -64,6 +64,16 @@ type TestResult = { timeUs?: number; }; +type V3TestResult = { + name: string; + resultV3Output: string; + mlirProgram: string; + resultV3Matched: boolean; + mlirMatched: boolean; + error?: string; + timeUs?: number; +}; + const API_BASE = "/api"; const SIDEBAR_MIN_WIDTH = 200; const SIDEBAR_MAX_WIDTH = 820; @@ -118,10 +128,11 @@ export default function App() { const [selected, setSelected] = React.useState(null); const [results, setResults] = React.useState>({}); const [remoteResults, setRemoteResults] = React.useState>({}); + const [v3Results, setV3Results] = React.useState>({}); const [search, setSearch] = React.useState(""); const [loading, setLoading] = React.useState(false); const [error, setError] = React.useState(null); - const [confirmTarget, setConfirmTarget] = React.useState<"plan" | "result" | "resultJson" | null>(null); + const [confirmTarget, setConfirmTarget] = React.useState<"plan" | "result" | "resultJson" | "mlir" | null>(null); const [nameDraft, setNameDraft] = React.useState(""); const [queryDraft, setQueryDraft] = React.useState(""); const [isEditingQuery, setIsEditingQuery] = React.useState(false); @@ -144,6 +155,7 @@ export default function App() { const [parsedRemoteResult, setParsedRemoteResult] = React.useState(null); const [parsedExpectedResult, setParsedExpectedResult] = React.useState(null); const [parsedMainResult, setParsedMainResult] = React.useState(null); + const [parsedV3Result, setParsedV3Result] = React.useState(null); const [disabledReasonDraft, setDisabledReasonDraft] = React.useState(""); const [shareNotice, setShareNotice] = React.useState(null); const shareTimerRef = React.useRef(null); @@ -152,13 +164,17 @@ export default function App() { const [expectedPlan, setExpectedPlan] = React.useState(""); const [expectedResult, setExpectedResult] = React.useState(""); const [expectedResultJson, setExpectedResultJson] = React.useState(""); + const [expectedMlir, setExpectedMlir] = React.useState(""); const [mainPlan, setMainPlan] = React.useState(""); const [mainResult, setMainResult] = React.useState(""); const [mainResultJson, setMainResultJson] = React.useState(""); + const [mainMlir, setMainMlir] = React.useState(""); const [resultTab, setResultTab] = React.useState<"actual" | "expected" | "main">("actual"); const [planTab, setPlanTab] = React.useState<"actual" | "expected" | "main">("actual"); const [jsonTab, setJsonTab] = React.useState<"actual" | "expected" | "main">("actual"); const [remoteResultTab, setRemoteResultTab] = React.useState<"actual" | "expected" | "main">("actual"); + const [resultV3Tab, setResultV3Tab] = React.useState<"actual" | "expected" | "main">("actual"); + const [mlirTab, setMlirTab] = React.useState<"actual" | "expected" | "main">("actual"); const loadTests = React.useCallback(async (preferName?: string) => { try { @@ -209,6 +225,8 @@ export default function App() { setPlanTab("actual"); setJsonTab("actual"); setRemoteResultTab("actual"); + setResultV3Tab("actual"); + setMlirTab("actual"); }, [selected]); React.useEffect(() => { @@ -216,9 +234,11 @@ export default function App() { setExpectedPlan(""); setExpectedResult(""); setExpectedResultJson(""); + setExpectedMlir(""); setMainPlan(""); setMainResult(""); setMainResultJson(""); + setMainMlir(""); return; } let active = true; @@ -229,18 +249,21 @@ export default function App() { setExpectedPlan(typeof data?.plan === "string" ? data.plan : ""); setExpectedResult(typeof data?.result === "string" ? data.result : ""); setExpectedResultJson(typeof data?.resultJson === "string" ? data.resultJson : ""); + setExpectedMlir(typeof data?.mlir === "string" ? data.mlir : ""); }) .catch(() => { if (!active) return; setExpectedPlan(""); setExpectedResult(""); setExpectedResultJson(""); + setExpectedMlir(""); }); const mainExpect = selected.mainVersion?.expect ?? {}; if (typeof mainExpect.plan === "string" || typeof mainExpect.result === "string") { setMainPlan(typeof mainExpect.plan === "string" ? mainExpect.plan : ""); setMainResult(typeof mainExpect.result === "string" ? mainExpect.result : ""); setMainResultJson(typeof mainExpect.resultJson === "string" ? mainExpect.resultJson : ""); + setMainMlir(typeof mainExpect.mlir === "string" ? mainExpect.mlir : ""); } else { fetch(`${API_BASE}/main?name=${encodeURIComponent(selected.name)}`) .then((res) => (res.ok ? res.json() : null)) @@ -249,12 +272,14 @@ export default function App() { setMainPlan(typeof data?.plan === "string" ? data.plan : ""); setMainResult(typeof data?.result === "string" ? data.result : ""); setMainResultJson(typeof data?.resultJson === "string" ? data.resultJson : ""); + setMainMlir(typeof data?.mlir === "string" ? data.mlir : ""); }) .catch(() => { if (!active) return; setMainPlan(""); setMainResult(""); setMainResultJson(""); + setMainMlir(""); }); } return () => { @@ -281,6 +306,15 @@ export default function App() { [] ); + // A test is only a v3 pass when the IR output reproduces the v2 output + // (expect.result). The MLIR program match is a separate, informational signal + // shown in its own panel and does not gate the overall pass/fail. + const isV3Pass = React.useCallback( + (result: V3TestResult) => + result.resultV3Matched === true, + [] + ); + const readApiErrorMessage = React.useCallback(async (res: Response, fallback: string) => { const payload = (await res.json().catch(() => null)) as { error?: unknown; details?: unknown } | null; if (typeof payload?.error === "string" && payload.error.trim()) { @@ -314,12 +348,19 @@ export default function App() { setError(null); setResults((prev) => ({ ...prev })); try { - const data = await fetchApiJson( - `${API_BASE}/run?test=${encodeURIComponent(name)}`, - "Failed to run test" - ); - setResults((prev) => ({ ...prev, [data.name]: data })); - if (!isLocalPass(data)) { + const [localData, v3Data] = await Promise.all([ + fetchApiJson( + `${API_BASE}/run?test=${encodeURIComponent(name)}`, + "Failed to run test" + ), + fetchApiJson( + `${API_BASE}/run-v3?test=${encodeURIComponent(name)}`, + "Failed to run v3 test" + ) + ]); + setResults((prev) => ({ ...prev, [localData.name]: localData })); + setV3Results((prev) => ({ ...prev, [v3Data.name]: v3Data })); + if (!isLocalPass(localData)) { showFailToast("1 test failed"); } } catch (err) { @@ -334,10 +375,12 @@ export default function App() { setError(null); setResults((prev) => ({ ...prev })); setRemoteResults((prev) => ({ ...prev })); + setV3Results((prev) => ({ ...prev })); try { - const [localData, remoteData] = await Promise.all([ + const [localData, remoteData, v3Data] = await Promise.all([ fetchApiJson(`${API_BASE}/run-all`, "Failed to run all tests"), - fetchApiJson(`${API_BASE}/run-all-remote`, "Failed to run all remote tests") + fetchApiJson(`${API_BASE}/run-all-remote`, "Failed to run all remote tests"), + fetchApiJson(`${API_BASE}/run-all-v3`, "Failed to run all v3 tests") ]); const nextLocal: Record = {}; for (const entry of localData) { @@ -347,10 +390,16 @@ export default function App() { for (const entry of remoteData) { nextRemote[entry.name] = entry; } + const nextV3: Record = {}; + for (const entry of v3Data) { + nextV3[entry.name] = entry; + } setResults(nextLocal); setRemoteResults(nextRemote); + setV3Results(nextV3); const localFailed = localData.filter((entry) => !isLocalPass(entry)).length; const remoteFailed = remoteData.filter((entry) => !isRemotePass(entry)).length; + const v3Failed = v3Data.filter((entry) => !isV3Pass(entry)).length; const notices: string[] = []; if (localFailed > 0) { notices.push(`${localFailed} local test${localFailed === 1 ? "" : "s"} failed`); @@ -358,6 +407,9 @@ export default function App() { if (remoteFailed > 0) { notices.push(`${remoteFailed} remote test${remoteFailed === 1 ? "" : "s"} failed`); } + if (v3Failed > 0) { + notices.push(`${v3Failed} v3 test${v3Failed === 1 ? "" : "s"} failed`); + } if (notices.length > 0) { showFailToast(notices.join(" / ")); } @@ -368,15 +420,26 @@ export default function App() { } }; - const acceptOutputs = async (target: "plan" | "result" | "resultJson") => { - if (!selected || !selectedResult) return; + const acceptOutputs = async (target: "plan" | "result" | "resultJson" | "mlir") => { + if (!selected) return; + const isV3Target = target === "mlir"; + const localResult = selectedResult; + const v3Result = selectedV3Result; + if (isV3Target ? !v3Result : !localResult) return; setLoading(true); setError(null); try { - const payload: { name: string; plan?: string; result?: string; resultJson?: string } = { name: selected.name }; - if (target === "plan") payload.plan = selectedResult.planOutput; - if (target === "result") payload.result = selectedResult.resultOutput; - if (target === "resultJson") payload.resultJson = selectedResult.resultJsonOutput; + const payload: { + name: string; + plan?: string; + result?: string; + resultJson?: string; + mlir?: string; + } = { name: selected.name }; + if (target === "plan" && localResult) payload.plan = localResult.planOutput; + if (target === "result" && localResult) payload.result = localResult.resultOutput; + if (target === "resultJson" && localResult) payload.resultJson = localResult.resultJsonOutput; + if (target === "mlir" && v3Result) payload.mlir = v3Result.mlirProgram; const res = await fetch(`${API_BASE}/update`, { method: "POST", headers: { "content-type": "application/json" }, @@ -390,17 +453,30 @@ export default function App() { : "Failed to update test"; throw new Error(message); } - setResults((prev) => ({ - ...prev, - [selected.name]: { - ...selectedResult, - planMatched: target === "plan" ? true : selectedResult.planMatched, - resultMatched: target === "result" ? true : selectedResult.resultMatched, - resultJsonMatched: target === "resultJson" ? true : selectedResult.resultJsonMatched + if (isV3Target && v3Result) { + setV3Results((prev) => ({ + ...prev, + [selected.name]: { + ...v3Result, + mlirMatched: target === "mlir" ? true : v3Result.mlirMatched + } + })); + if (target === "mlir") { + setExpectedMlir(v3Result.mlirProgram); + } + } else if (localResult) { + setResults((prev) => ({ + ...prev, + [selected.name]: { + ...localResult, + planMatched: target === "plan" ? true : localResult.planMatched, + resultMatched: target === "result" ? true : localResult.resultMatched, + resultJsonMatched: target === "resultJson" ? true : localResult.resultJsonMatched + } + })); + if (target === "resultJson") { + setExpectedResultJson(localResult.resultJsonOutput ?? ""); } - })); - if (target === "resultJson") { - setExpectedResultJson(selectedResult.resultJsonOutput ?? ""); } } catch (err) { const message = err instanceof Error ? err.message : "Failed to update test JSON."; @@ -840,6 +916,7 @@ export default function App() { const selectedResult = selected ? results[selected.name] : undefined; const selectedRemoteResult = selected ? remoteResults[selected.name] : undefined; + const selectedV3Result = selected ? v3Results[selected.name] : undefined; const getTestRunStatus = React.useCallback( (test: TestMeta) => { if (!test.enabled) return "disabled" as const; @@ -848,9 +925,11 @@ export default function App() { if (!isLocalPass(localResult)) return "fail" as const; const remoteResult = remoteResults[test.name]; if (remoteResult && !isRemotePass(remoteResult)) return "fail" as const; + const v3Result = v3Results[test.name]; + if (v3Result && !isV3Pass(v3Result)) return "fail" as const; return "pass" as const; }, - [isLocalPass, isRemotePass, remoteResults, results] + [isLocalPass, isRemotePass, isV3Pass, remoteResults, results, v3Results] ); const allTags = React.useMemo(() => { const set = new Set(); @@ -1083,7 +1162,8 @@ export default function App() { setParsedRemoteResult(parseCsv(selectedRemoteResult?.resultOutput ?? "")); setParsedExpectedResult(parseCsv(expectedResult)); setParsedMainResult(parseCsv(mainResult)); - }, [selectedRemoteResult?.resultOutput, selectedResult?.resultOutput, expectedResult, mainResult]); + setParsedV3Result(parseCsv(selectedV3Result?.resultV3Output ?? "")); + }, [selectedRemoteResult?.resultOutput, selectedResult?.resultOutput, selectedV3Result?.resultV3Output, expectedResult, mainResult]); const renderTableResult = React.useCallback((rows: string[][]) => (
@@ -1241,11 +1321,13 @@ export default function App() { {visibleTests.map((test) => { const testResult = results[test.name]; const remoteResult = remoteResults[test.name]; + const v3Result = v3Results[test.name]; const status = getTestRunStatus(test); const isPending = status === "pending"; const isDisabled = status === "disabled"; const isPass = status === "pass"; const remoteFailed = !!remoteResult && !isRemotePass(remoteResult); + const v3Failed = !!v3Result && !isV3Pass(v3Result); const statusClass = isDisabled ? "text-amber-400" : isPass @@ -1291,7 +1373,9 @@ export default function App() { ? timingLabel ?? "pass" : remoteFailed ? "remote fail" - : "fail"} + : v3Failed + ? "ir fail" + : "fail"} @@ -1858,6 +1942,132 @@ export default function App() {
)} + {selected && ( +
+
+
+

V3 (MLIR) Query Output

+

+ Run All also runs this test through the QueryInterpreterV3 MLIR path. Its result is + compared against the same expect.result as the local suite; the emitted DB MLIR program is compared + against expect.mlir. +

+
+
+ + {selectedV3Result ? ( + <> +
+
+

V3 Result Output

+
+
+ + + +
+ + {selectedV3Result.resultV3Matched ? "match" : "mismatch"} + + {typeof selectedV3Result.timeUs === "number" && ( + + {selectedV3Result.timeUs} us + + )} +
+
+ {resultV3Tab === "actual" + ? renderCsvOrText(parsedV3Result, selectedV3Result.resultV3Output) + : resultV3Tab === "expected" + ? renderCsvOrText(parsedExpectedResult, expectedResult) + : renderCsvOrText(parsedMainResult, mainResult)} +
+ +
+
+

MLIR Program

+
+
+ + + +
+ + {selectedV3Result.mlirMatched ? "match" : "mismatch"} + + {!selectedV3Result.mlirMatched && ( + + )} +
+
+ {renderTextResult( + mlirTab === "actual" + ? selectedV3Result.mlirProgram + : mlirTab === "expected" + ? expectedMlir + : mainMlir + )} +
+ + ) : ( +
+ Use Run All to populate v3 output for this test. +
+ )} +
+ )} + {(!selected || !selectedResult) && !loading && !error && (
{selected @@ -1872,7 +2082,7 @@ export default function App() {

Update test expectations?

- This will overwrite the expected {confirmTarget === "resultJson" ? "JSON result" : confirmTarget} in the JSON file for{" "} + This will overwrite the expected {confirmTarget === "resultJson" ? "JSON result" : confirmTarget === "mlir" ? "MLIR program" : confirmTarget} in the JSON file for{" "} {selected?.name}.