From f4a248a5723b424bbb83cf9bd46fc9b7dcf6210b Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Sat, 21 Feb 2026 16:37:03 -0500 Subject: [PATCH] fix: resolve WASM marimo test failure caused by fastparquet unavailability in Pyodide The WASM Playwright test timed out because micropip.install("buckaroo") failed on fastparquet (C extension with no pure-Python wheel for Pyodide). This cascaded to all downstream cells, preventing widget rendering. Three-part fix: - WASM notebooks: install deps manually with deps=False, create a minimal fastparquet stub module so buckaroo's import chain succeeds - serialization_utils.py: make fastparquet import optional (try/except) for forward compatibility when the next release is published to PyPI - Test: replace fixed-timeout waitFor with polling loop, increase timeout to 300s, add console log capture for CI debugging Co-Authored-By: Claude Opus 4.6 --- buckaroo/serialization_utils.py | 37 +++-- .../marimo-wasm/buckaroo_ddd_tour.py | 29 +++- .../marimo-wasm/buckaroo_ddd_tour_full.py | 29 +++- .../marimo-wasm/buckaroo_simple.py | 27 +++- .../playwright.config.wasm-marimo.ts | 4 +- .../pw-tests/wasm-marimo.spec.ts | 134 ++++++++++++++---- 6 files changed, 212 insertions(+), 48 deletions(-) diff --git a/buckaroo/serialization_utils.py b/buckaroo/serialization_utils.py index 97660c34d..571b8f0fe 100644 --- a/buckaroo/serialization_utils.py +++ b/buckaroo/serialization_utils.py @@ -5,9 +5,13 @@ from typing import Dict, Any, List, Tuple from pandas._libs.tslibs import timezones from pandas.core.dtypes.dtypes import DatetimeTZDtype -from fastparquet import json as fp_json import logging +try: + from fastparquet import json as fp_json +except ImportError: + fp_json = None # fastparquet not available (e.g. Pyodide/WASM) + from buckaroo.df_util import old_col_new_col, to_chars logger = logging.getLogger() @@ -128,19 +132,22 @@ def pd_to_obj(df:pd.DataFrame) -> Dict[str, Any]: pass -class MyJsonImpl(fp_json.BaseImpl): - def __init__(self): - pass - #for some reason the following line causes errors, so I have to reimport ujson_dumps - # from pandas._libs.json import ujson_dumps - # self.dumps = ujson_dumps +if fp_json is not None: + class MyJsonImpl(fp_json.BaseImpl): + def __init__(self): + pass + #for some reason the following line causes errors, so I have to reimport ujson_dumps + # from pandas._libs.json import ujson_dumps + # self.dumps = ujson_dumps - def dumps(self, data): - from pandas._libs.json import ujson_dumps - return ujson_dumps(data, default_handler=str).encode("utf-8") + def dumps(self, data): + from pandas._libs.json import ujson_dumps + return ujson_dumps(data, default_handler=str).encode("utf-8") - def loads(self, s): - return self.api.loads(s) + def loads(self, s): + return self.api.loads(s) +else: + MyJsonImpl = None # type: ignore[assignment,misc] def get_multiindex_to_cols_sers(index) -> List[Tuple[str, Any]]: #pd.Series[Any] if not isinstance(index, pd.MultiIndex): @@ -168,6 +175,12 @@ def prepare_df_for_serialization(df:pd.DataFrame) -> pd.DataFrame: return df2 def to_parquet(df): + if fp_json is None: + raise ImportError( + "fastparquet is required for parquet serialization but is not installed. " + "This is expected in Pyodide/WASM environments." + ) + data: BytesIO = BytesIO() # data.close doesn't work in pyodide, so we make close a no-op diff --git a/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour.py b/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour.py index 1d594349f..5d48f2d4f 100644 --- a/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour.py +++ b/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour.py @@ -335,10 +335,31 @@ async def _(): if "pyodide" in sys.modules: # a hacky way to figure out if we're running in pyodide import micropip - - # keep_going=True allows micropip to skip packages without pure-Python wheels (e.g., fastparquet) - # and install what it can. Buckaroo will work without fastparquet in WASM. - await micropip.install("buckaroo", keep_going=True) + import types + + # Install buckaroo's pure-Python dependencies first, skipping fastparquet + # (C extension, not available in WASM). Then install buckaroo with deps=False. + await micropip.install( + ["anywidget", "graphlib_backport", "cloudpickle"], + keep_going=True, + ) + await micropip.install("buckaroo", deps=False) + + # Create a minimal fastparquet stub so buckaroo can import. + # buckaroo.serialization_utils imports fastparquet.json at module level; + # the parquet serialization path won't work in WASM, but JSON fallback does. + if "fastparquet" not in sys.modules: + _fp = types.ModuleType("fastparquet") + _fp_json = types.ModuleType("fastparquet.json") + + class _StubBaseImpl: + pass + + _fp_json.BaseImpl = _StubBaseImpl + _fp_json._get_cached_codec = lambda: None + _fp.json = _fp_json + sys.modules["fastparquet"] = _fp + sys.modules["fastparquet.json"] = _fp_json import buckaroo from buckaroo import BuckarooInfiniteWidget diff --git a/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour_full.py b/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour_full.py index 1d594349f..5d48f2d4f 100644 --- a/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour_full.py +++ b/docs/example-notebooks/marimo-wasm/buckaroo_ddd_tour_full.py @@ -335,10 +335,31 @@ async def _(): if "pyodide" in sys.modules: # a hacky way to figure out if we're running in pyodide import micropip - - # keep_going=True allows micropip to skip packages without pure-Python wheels (e.g., fastparquet) - # and install what it can. Buckaroo will work without fastparquet in WASM. - await micropip.install("buckaroo", keep_going=True) + import types + + # Install buckaroo's pure-Python dependencies first, skipping fastparquet + # (C extension, not available in WASM). Then install buckaroo with deps=False. + await micropip.install( + ["anywidget", "graphlib_backport", "cloudpickle"], + keep_going=True, + ) + await micropip.install("buckaroo", deps=False) + + # Create a minimal fastparquet stub so buckaroo can import. + # buckaroo.serialization_utils imports fastparquet.json at module level; + # the parquet serialization path won't work in WASM, but JSON fallback does. + if "fastparquet" not in sys.modules: + _fp = types.ModuleType("fastparquet") + _fp_json = types.ModuleType("fastparquet.json") + + class _StubBaseImpl: + pass + + _fp_json.BaseImpl = _StubBaseImpl + _fp_json._get_cached_codec = lambda: None + _fp.json = _fp_json + sys.modules["fastparquet"] = _fp + sys.modules["fastparquet.json"] = _fp_json import buckaroo from buckaroo import BuckarooInfiniteWidget diff --git a/docs/example-notebooks/marimo-wasm/buckaroo_simple.py b/docs/example-notebooks/marimo-wasm/buckaroo_simple.py index b862275da..b27fce284 100644 --- a/docs/example-notebooks/marimo-wasm/buckaroo_simple.py +++ b/docs/example-notebooks/marimo-wasm/buckaroo_simple.py @@ -9,10 +9,35 @@ async def _(): import marimo as mo import pandas as pd import sys + import types if "pyodide" in sys.modules: import micropip - await micropip.install("buckaroo", keep_going=True) + + # Install buckaroo's dependencies that work in Pyodide first, + # explicitly skipping fastparquet (C extension, not available in WASM). + # Then install buckaroo itself with deps=False. + await micropip.install( + ["anywidget", "graphlib_backport", "cloudpickle"], + keep_going=True, + ) + await micropip.install("buckaroo", deps=False) + + # Create a minimal fastparquet stub so buckaroo can import. + # buckaroo.serialization_utils imports fastparquet.json at module level; + # the parquet serialization path won't work in WASM, but JSON fallback does. + if "fastparquet" not in sys.modules: + _fp = types.ModuleType("fastparquet") + _fp_json = types.ModuleType("fastparquet.json") + + class _StubBaseImpl: + pass + + _fp_json.BaseImpl = _StubBaseImpl + _fp_json._get_cached_codec = lambda: None + _fp.json = _fp_json + sys.modules["fastparquet"] = _fp + sys.modules["fastparquet.json"] = _fp_json import buckaroo from buckaroo import BuckarooWidget, BuckarooInfiniteWidget diff --git a/packages/buckaroo-js-core/playwright.config.wasm-marimo.ts b/packages/buckaroo-js-core/playwright.config.wasm-marimo.ts index eafd857b5..521f056a8 100644 --- a/packages/buckaroo-js-core/playwright.config.wasm-marimo.ts +++ b/packages/buckaroo-js-core/playwright.config.wasm-marimo.ts @@ -19,8 +19,8 @@ export default defineConfig({ trace: 'on-first-retry', ...devices['Desktop Chrome'], }, - // Longer timeout for WASM: Pyodide initialization can be slow (15-30s) - timeout: 60_000, + // Longer timeout for WASM: Pyodide init + micropip install + cell execution + timeout: 300_000, projects: [ { diff --git a/packages/buckaroo-js-core/pw-tests/wasm-marimo.spec.ts b/packages/buckaroo-js-core/pw-tests/wasm-marimo.spec.ts index 2fc58249e..f8e524208 100644 --- a/packages/buckaroo-js-core/pw-tests/wasm-marimo.spec.ts +++ b/packages/buckaroo-js-core/pw-tests/wasm-marimo.spec.ts @@ -1,40 +1,124 @@ -import { test, expect, Page } from '@playwright/test'; +import { test, expect } from '@playwright/test'; /** * Single smoke test for Buckaroo rendering in marimo WASM (Pyodide). * + * The WASM page is generated from docs/example-notebooks/marimo-wasm/buckaroo_simple.py + * using: bash scripts/marimo_wasm_output.sh buckaroo_simple.py run + * * Full test suite saved in https://github.com/buckaroo-data/buckaroo/issues/513 * for re-enabling once WASM test infrastructure is more stable. */ -let sharedPage: Page; - -test.describe('Buckaroo in Marimo WASM (Pyodide)', () => { - test.describe.configure({ mode: 'serial' }); +test('Buckaroo WASM marimo - page loads and widgets render', async ({ page }) => { + // Pyodide init + micropip install buckaroo + cell execution can take 2-3 min + test.setTimeout(300_000); - test.beforeAll(async ({ browser }) => { - sharedPage = await browser.newPage(); - await sharedPage.goto('/'); - // Wait for Pyodide init + buckaroo widget + AG-Grid render - await sharedPage.locator('.buckaroo_anywidget').first().waitFor({ state: 'visible', timeout: 60_000 }); - await sharedPage.locator('.ag-cell').first().waitFor({ state: 'visible', timeout: 15_000 }); + // Capture console messages for debugging failures + const consoleLogs: string[] = []; + page.on('console', (msg) => { + consoleLogs.push(`[${msg.type()}] ${msg.text()}`); }); - - test.afterAll(async () => { - await sharedPage?.close(); + page.on('pageerror', (err) => { + consoleLogs.push(`[PAGE_ERROR] ${err.message}`); }); - test('page loads and WASM widgets render with data', async () => { - // At least one buckaroo widget rendered - const widgets = await sharedPage.locator('.buckaroo_anywidget').all(); - expect(widgets.length).toBeGreaterThanOrEqual(1); + await page.goto('/'); - // AG-Grid cells are visible (data actually rendered) - const cells = await sharedPage.locator('.ag-cell').all(); - expect(cells.length).toBeGreaterThan(0); + // Wait for marimo cells to render - poll for visible notebook content. + // The "Buckaroo in Marimo WASM" heading appears once cells execute. + const pollInterval = 5_000; + const maxWait = 240_000; + let elapsed = 0; + let contentRendered = false; - // Column headers are present - const headers = await sharedPage.locator('.ag-header-cell-text').all(); - expect(headers.length).toBeGreaterThan(0); - }); + while (elapsed < maxWait) { + await page.waitForTimeout(pollInterval); + elapsed += pollInterval; + + const pageText = await page.evaluate(() => document.body.innerText); + if (pageText.includes('Buckaroo in Marimo WASM')) { + contentRendered = true; + break; + } + } + + if (!contentRendered) { + // Dump console logs to help diagnose CI failures + console.log('=== Console logs (last 80) ==='); + for (const log of consoleLogs.slice(-80)) { + console.log(log); + } + console.log('=== End console logs ==='); + } + + expect(contentRendered, 'Marimo notebook content should have rendered').toBe(true); + + // Wait for buckaroo widgets to appear after cell execution. + // The widget renders after micropip installs buckaroo and cells run. + const widgetMaxWait = 60_000; + let widgetElapsed = 0; + let widgetFound = false; + + while (widgetElapsed < widgetMaxWait) { + await page.waitForTimeout(5_000); + widgetElapsed += 5_000; + + // Check in main page + if ((await page.locator('.buckaroo_anywidget').count()) > 0) { + widgetFound = true; + break; + } + + // Also check inside iframes (anywidget may render in iframes) + for (const frame of page.frames()) { + if (frame === page.mainFrame()) continue; + try { + if ((await frame.locator('.buckaroo_anywidget').count()) > 0) { + widgetFound = true; + break; + } + } catch (_e) { + // frame may be detached + } + } + if (widgetFound) break; + } + + if (!widgetFound) { + // Dump console logs to help diagnose CI failures + console.log('=== Console logs (last 80) ==='); + for (const log of consoleLogs.slice(-80)) { + console.log(log); + } + console.log('=== End console logs ==='); + } + + // Check for buckaroo widget and AG-Grid content in main page and frames + let agCellFound = false; + let headerFound = false; + + const mainBuckaroo = await page.locator('.buckaroo_anywidget').count(); + const mainAgCell = await page.locator('.ag-cell').count(); + const mainHeaders = await page.locator('.ag-header-cell-text').count(); + + if (mainBuckaroo > 0) widgetFound = true; + if (mainAgCell > 0) agCellFound = true; + if (mainHeaders > 0) headerFound = true; + + // Check all frames as fallback + for (const frame of page.frames()) { + if (frame === page.mainFrame()) continue; + try { + if (!widgetFound && (await frame.locator('.buckaroo_anywidget').count()) > 0) widgetFound = true; + if (!agCellFound && (await frame.locator('.ag-cell').count()) > 0) agCellFound = true; + if (!headerFound && (await frame.locator('.ag-header-cell-text').count()) > 0) headerFound = true; + } catch (_e) { + // frame may be detached + } + } + + expect(widgetFound, 'At least one .buckaroo_anywidget should be present').toBe(true); + expect(agCellFound, 'AG-Grid cells should be visible (data rendered)').toBe(true); + expect(headerFound, 'AG-Grid column headers should be visible').toBe(true); });