From 8aa867a4bbc2b965b78ea7c71baf4fef7fd90992 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:49:34 +0300 Subject: [PATCH 01/27] feat(grpc-web): add Pyodide/WASM grpc-web transport Let the async client's gRPC data path run under Pyodide/WebAssembly (marimo, browser), where grpcio has no wheel and raw sockets are unavailable. Base changes (no-ops on normal platforms): - setup.cfg: mark grpcio with `; sys_platform != "emscripten"` so micropip skips it under Pyodide while CPython installs it unchanged. - weaviate/proto/v1/__init__.py: when grpcio distribution metadata is absent, fall back to version 1.72.1 so a working generated-proto variant is selected. Restricted to grpcio; a missing protobuf still surfaces. New companion package packages/grpc-web (weaviate-python-grpc-web): - A sys.modules `grpc` shim that satisfies the client's import-time grpc surface and the `grpc.aio.Channel` / awaitability contracts; installs itself only under Emscripten so it never clobbers a real grpcio. - GrpcWebChannel: frames unary RPCs as grpc-web and POSTs them via pyodide pyfetch (with an httpx sender for CPython tests); folds call metadata into fetch headers; maps grpc-web trailers/status to the client's error types. - Reuses the client's generated protobuf stubs (no codegen fork). Async-only; bidirectional BatchStream is intentionally unsupported over fetch. Tests (25 passing): grpc-web framing, transport round-trips and error mapping, a subprocess test that imports weaviate under the shim with a real-proto unary round trip, and base proto-guard regression tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/grpc-web/README.md | 57 ++++ packages/grpc-web/pyproject.toml | 30 ++ .../src/weaviate_grpc_web/__init__.py | 51 ++++ .../src/weaviate_grpc_web/_channel.py | 233 +++++++++++++++ .../src/weaviate_grpc_web/_framing.py | 68 +++++ .../grpc-web/src/weaviate_grpc_web/_sender.py | 59 ++++ .../grpc-web/src/weaviate_grpc_web/_shim.py | 265 ++++++++++++++++++ .../grpc-web/src/weaviate_grpc_web/py.typed | 0 packages/grpc-web/tests/conftest.py | 7 + packages/grpc-web/tests/test_framing.py | 59 ++++ packages/grpc-web/tests/test_shim_install.py | 111 ++++++++ packages/grpc-web/tests/test_transport.py | 158 +++++++++++ proto_test/test_proto.py | 31 +- setup.cfg | 2 +- weaviate/proto/v1/__init__.py | 20 +- 15 files changed, 1146 insertions(+), 5 deletions(-) create mode 100644 packages/grpc-web/README.md create mode 100644 packages/grpc-web/pyproject.toml create mode 100644 packages/grpc-web/src/weaviate_grpc_web/__init__.py create mode 100644 packages/grpc-web/src/weaviate_grpc_web/_channel.py create mode 100644 packages/grpc-web/src/weaviate_grpc_web/_framing.py create mode 100644 packages/grpc-web/src/weaviate_grpc_web/_sender.py create mode 100644 packages/grpc-web/src/weaviate_grpc_web/_shim.py create mode 100644 packages/grpc-web/src/weaviate_grpc_web/py.typed create mode 100644 packages/grpc-web/tests/conftest.py create mode 100644 packages/grpc-web/tests/test_framing.py create mode 100644 packages/grpc-web/tests/test_shim_install.py create mode 100644 packages/grpc-web/tests/test_transport.py diff --git a/packages/grpc-web/README.md b/packages/grpc-web/README.md new file mode 100644 index 000000000..5810e9f06 --- /dev/null +++ b/packages/grpc-web/README.md @@ -0,0 +1,57 @@ +# weaviate-python-grpc-web + +A grpc-web / WebAssembly (Pyodide) transport for the +[Weaviate Python client](https://github.com/weaviate/weaviate-python-client), so the +client's **async** gRPC data path can run inside a browser (marimo notebooks, Pyodide, +WASM workers) where there is no socket and no `grpcio` wheel. + +It is built from the same repository as `weaviate-client` and reuses its generated +protobuf stubs — it does **not** fork code generation. + +## How it works + +Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard-imports +`grpc` at module load. This package installs a small pure-Python `grpc` shim into +`sys.modules` **before** `import weaviate`, which: + +- satisfies every import-time `import grpc` / `from grpc(.aio) import ...` in the base + client and its generated `*_pb2_grpc` stubs; +- provides `grpc.aio.Channel` as a real base class, so the grpc-web channel + (`GrpcWebChannel`) subclasses it and the client's `isinstance(..., grpc.aio.Channel)` + assertions pass; +- satisfies the generated v6300 stub's version gate + (`grpc.__version__` / `grpc._utilities.first_version_is_lower`). + +The `GrpcWebChannel` frames unary RPCs as grpc-web (a 5-byte header + protobuf payload) +and POSTs them via `pyodide.http.pyfetch` to a server fronted by a grpc-web transcoder +(e.g. Envoy or [connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)). Call +metadata (API key / OIDC bearer) is folded into `fetch` headers. + +## Usage + +```python +import weaviate_grpc_web # installs the grpc shim under Emscripten (no-op elsewhere) +import weaviate + +client = weaviate.use_async_with_local(skip_init_checks=True) +await client.connect() +collection = client.collections.get("Article") +await collection.query.near_text("hello", limit=3) +``` + +## Supported / unsupported + +| RPC | Kind | Status | +|----------------------------------------------------------|-----------------|--------| +| Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary | ✅ works over grpc-web | +| Health check (`/grpc.health.v1.Health/Check`) | unary | ✅ (recommend `skip_init_checks=True` + REST `/.well-known/ready`) | +| References (`/batch/references`) | REST | ✅ via httpx-in-Pyodide | +| `batch.stream()` / `batch.experimental()` (BatchStream) | bidi streaming | ❌ not possible over grpc-web/fetch — use `insert_many()` / `batch.dynamic()` / `fixed_size()` / `rate_limit()` | +| Synchronous client | — | ❌ async-only under WASM | + +## Testing on CPython + +`weaviate_grpc_web.install(force=True)` installs the shim on a normal CPython +interpreter (run it in a fresh process, before importing `weaviate`). Inject a sender +with `weaviate_grpc_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the +transport against an Envoy/vanguard transcoder without a browser. diff --git a/packages/grpc-web/pyproject.toml b/packages/grpc-web/pyproject.toml new file mode 100644 index 000000000..5cadabf35 --- /dev/null +++ b/packages/grpc-web/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=65", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "weaviate-python-grpc-web" +description = "grpc-web / WASM (Pyodide) transport for the Weaviate Python client" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "BSD-3-Clause" } +authors = [{ name = "Weaviate", email = "hello@weaviate.io" }] +keywords = ["weaviate", "grpc-web", "pyodide", "wasm", "emscripten"] +# Version is kept in lockstep with weaviate-client. TODO(lockstep): derive from the same +# git tag via setuptools_scm and assert the built versions match in CI before publishing. +version = "0.0.1.dev0" +# Deliberately depends on weaviate-client WITHOUT grpcio (grpcio is excluded under +# Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). +dependencies = [ + "weaviate-client", +] + +[project.urls] +Source = "https://github.com/weaviate/weaviate-python-client" +Tracker = "https://github.com/weaviate/weaviate-python-client/issues" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +weaviate_grpc_web = ["py.typed"] diff --git a/packages/grpc-web/src/weaviate_grpc_web/__init__.py b/packages/grpc-web/src/weaviate_grpc_web/__init__.py new file mode 100644 index 000000000..79a6cb1d8 --- /dev/null +++ b/packages/grpc-web/src/weaviate_grpc_web/__init__.py @@ -0,0 +1,51 @@ +"""grpc-web / WASM transport for the Weaviate Python client. + +Under Pyodide/Emscripten there is no ``grpcio`` wheel. Importing this package installs a +pure-Python ``grpc`` shim into ``sys.modules`` (and forces the pure-Python protobuf +runtime) so that the subsequent ``import weaviate`` succeeds and its async gRPC data path +runs over grpc-web (``fetch``) instead of HTTP/2 sockets. + +Usage under Pyodide:: + + import weaviate_grpc_web # installs the grpc shim (no-op off Emscripten) + import weaviate + + client = weaviate.use_async_with_local(skip_init_checks=True) + await client.connect() + +The shim is installed automatically only under Emscripten, so importing this package on a +normal CPython install never clobbers a real, working ``grpcio``. Async clients only — +the synchronous client is not supported in the browser. +""" + +import os +import sys + +from ._shim import StatusCode, install, is_installed + +__all__ = [ + "install", + "is_installed", + "set_sender", + "make_httpx_sender", + "GrpcWebChannel", + "StatusCode", +] + + +def _bootstrap() -> None: + if sys.platform == "emscripten": + # The pure-Python protobuf runtime always works; the upb C-extension may not be + # present. Set before ``import weaviate`` (which imports protobuf) so it takes + # effect. ``setdefault`` lets a user override it explicitly. + os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") + install() + + +_bootstrap() + +# Imported after the bootstrap. These modules pull their grpc base classes directly from +# ``._shim`` (not via ``sys.modules['grpc']``), so importing them is safe regardless of +# whether the shim was installed. +from ._channel import GrpcWebChannel, set_sender # noqa: E402 +from ._sender import make_httpx_sender # noqa: E402 diff --git a/packages/grpc-web/src/weaviate_grpc_web/_channel.py b/packages/grpc-web/src/weaviate_grpc_web/_channel.py new file mode 100644 index 000000000..5e7b9bd55 --- /dev/null +++ b/packages/grpc-web/src/weaviate_grpc_web/_channel.py @@ -0,0 +1,233 @@ +"""The grpc-web channel and multicallables. + +:class:`GrpcWebChannel` implements the small slice of the ``grpc.aio`` channel interface +that ``weaviate``'s generated stub and ``ConnectionV4`` actually use — ``unary_unary``, +``stream_stream`` and ``close`` — by framing requests as grpc-web and POSTing them via a +pluggable async sender. It subclasses the shim's ``grpc.aio.Channel`` (:class:`AioChannel`) +so the ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py`` hold. + +Only unary RPCs are supported (Search, Aggregate, TenantsGet, BatchObjects, +BatchReferences, BatchDelete, and the unary health check). ``stream_stream`` (the bidi +``BatchStream`` used by opt-in server-side batching) cannot work over grpc-web/fetch and +raises a clear error. +""" + +import base64 +import urllib.parse +from typing import Any, Callable, Dict, Optional + +from ._framing import encode_message, split_response +from ._sender import Sender, pyfetch_sender +from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int + +# Module-level default sender; overridable for tests / non-browser runtimes. +_default_sender: Sender = pyfetch_sender + + +def set_sender(sender: Sender) -> None: + """Override the default async sender used by new channels (tests/integration).""" + global _default_sender + _default_sender = sender + + +def get_sender() -> Sender: + return _default_sender + + +def _encode_timeout(seconds: float) -> str: + """Encode a timeout as a grpc-timeout header value (````).""" + millis = max(1, int(seconds * 1000)) + if millis < 100_000_000: + return f"{millis}m" + return f"{max(1, int(seconds))}S" + + +def _fold_metadata(headers: Dict[str, str], metadata: Any) -> None: + """Fold gRPC call metadata (``[(key, value), ...]``) into fetch headers. + + Binary ``-bin`` keys are base64-encoded as grpc-web requires. + """ + if not metadata: + return + for key, value in metadata: + name = key.lower() + if name.endswith("-bin"): + raw = value if isinstance(value, (bytes, bytearray)) else str(value).encode() + headers[name] = base64.b64encode(raw).decode("ascii") + else: + headers[name] = value if isinstance(value, str) else str(value) + + +def _header_lookup(headers: Dict[str, str], name: str) -> Optional[str]: + target = name.lower() + for key, value in headers.items(): + if key.lower() == target: + return value + return None + + +class _UnaryUnaryMultiCallable: + """Awaitable multicallable bound by ``WeaviateStub.__init__``. + + Called as ``await mc(request, metadata=..., timeout=...)`` (and, for the health + check, as ``mc(request, timeout=...)`` with no metadata). + """ + + def __init__( + self, + channel: "GrpcWebChannel", + path: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + ) -> None: + self._channel = channel + self._path = path + self._serialize = request_serializer + self._deserialize = response_deserializer + + async def __call__( + self, + request: Any, + *, + metadata: Any = None, + timeout: Optional[float] = None, + credentials: Any = None, + wait_for_ready: Any = None, + compression: Any = None, + ) -> Any: + payload = self._serialize(request) + return await self._channel._unary(self._path, payload, self._deserialize, metadata, timeout) + + +class _UnsupportedStreamMultiCallable: + """Placeholder for ``stream_stream`` (bidirectional streaming). + + Calling it raises immediately, before the ``async for`` in ``connect/v4.py:1243`` + begins iterating. + """ + + def __init__(self, path: str) -> None: + self._path = path + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + raise RuntimeError( + f"Bidirectional streaming RPC {self._path!r} (server-side batching / " + "BatchStream) is not supported over grpc-web/fetch. Use insert_many(), or " + "batch.dynamic() / fixed_size() / rate_limit(), instead of batch.stream()." + ) + + +class GrpcWebChannel(AioChannel): + """grpc-web/fetch implementation of the async grpc channel slice the client uses.""" + + def __init__( + self, + target: Optional[str], + secure: bool, + options: Any = None, + sender: Optional[Sender] = None, + ) -> None: + if not target: + raise ValueError("GrpcWebChannel requires a target (host:port)") + scheme = "https" if secure else "http" + self._base_url = f"{scheme}://{target}" + self._sender: Sender = sender or get_sender() + + def unary_unary( + self, + method: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + _registered_method: bool = False, + ) -> _UnaryUnaryMultiCallable: + return _UnaryUnaryMultiCallable(self, method, request_serializer, response_deserializer) + + def stream_stream( + self, + method: str, + request_serializer: Callable[[Any], bytes], + response_deserializer: Callable[[bytes], Any], + _registered_method: bool = False, + ) -> _UnsupportedStreamMultiCallable: + return _UnsupportedStreamMultiCallable(method) + + async def close(self, grace: Optional[float] = None) -> None: + # Nothing to tear down: each call is an independent fetch. + return None + + async def _unary( + self, + path: str, + payload: bytes, + deserialize: Callable[[bytes], Any], + metadata: Any, + timeout: Optional[float], + ) -> Any: + headers: Dict[str, str] = { + "content-type": "application/grpc-web+proto", + "accept": "application/grpc-web+proto", + "x-grpc-web": "1", + "x-user-agent": "weaviate-python-grpc-web", + } + _fold_metadata(headers, metadata) + if timeout is not None: + headers["grpc-timeout"] = _encode_timeout(timeout) + + url = self._base_url + path + status, resp_headers, body = await self._sender( + url, headers, encode_message(payload), timeout + ) + return self._handle_response(status, resp_headers, body, deserialize) + + @staticmethod + def _handle_response( + http_status: int, + resp_headers: Dict[str, str], + body: bytes, + deserialize: Callable[[bytes], Any], + ) -> Any: + messages, trailers = split_response(body) if body else ([], {}) + + raw_status = trailers.get("grpc-status") + if raw_status is None: + raw_status = _header_lookup(resp_headers, "grpc-status") + raw_message = ( + trailers.get("grpc-message") or _header_lookup(resp_headers, "grpc-message") or "" + ) + message = urllib.parse.unquote(raw_message) + + if raw_status is None: + if http_status != 200: + raise AioRpcError( + code=_status_from_http(http_status), + details=f"HTTP {http_status} from grpc-web endpoint", + ) + code = StatusCode.OK + else: + code = status_from_int(int(raw_status)) + + if code is not StatusCode.OK: + raise AioRpcError(code=code, details=message) + if not messages: + raise AioRpcError( + code=StatusCode.INTERNAL, + details="grpc-web response contained no message frame", + ) + return deserialize(messages[0]) + + +def _status_from_http(http_status: int) -> StatusCode: + """Map an HTTP status to a gRPC status when no grpc-status is present. + + Mirrors the grpc-web spec's HTTP-to-gRPC code mapping. + """ + return { + 400: StatusCode.INTERNAL, + 401: StatusCode.UNAUTHENTICATED, + 403: StatusCode.PERMISSION_DENIED, + 404: StatusCode.UNIMPLEMENTED, + 429: StatusCode.UNAVAILABLE, + 502: StatusCode.UNAVAILABLE, + 503: StatusCode.UNAVAILABLE, + 504: StatusCode.UNAVAILABLE, + }.get(http_status, StatusCode.UNKNOWN) diff --git a/packages/grpc-web/src/weaviate_grpc_web/_framing.py b/packages/grpc-web/src/weaviate_grpc_web/_framing.py new file mode 100644 index 000000000..85b6f6972 --- /dev/null +++ b/packages/grpc-web/src/weaviate_grpc_web/_framing.py @@ -0,0 +1,68 @@ +r"""grpc-web binary framing (``application/grpc-web+proto``). + +A grpc-web message frame is a 1-byte flag + 4-byte big-endian length + payload: + + +--------+----------------+----------------------+ + | flag | length (uint32)| payload (length bytes)| + +--------+----------------+----------------------+ + +The flag's high bit (``0x80``) marks a trailer frame whose payload is an +HTTP/1-style header block (``grpc-status: 0\\r\\ngrpc-message: ...``). The low bit +(``0x01``) marks a compressed message, which this transport neither sends nor +accepts. A unary grpc-web response body is one or more message frames followed by +exactly one trailer frame (or a "trailers-only" response carrying the status in +the HTTP headers, handled by the caller). +""" + +import struct +from typing import Dict, Iterator, List, Tuple + +_FLAG_TRAILER = 0x80 +_FLAG_COMPRESSED = 0x01 +_HEADER = struct.Struct(">BI") # 1 flag byte + 4-byte big-endian length + + +def encode_message(payload: bytes) -> bytes: + """Frame a single (uncompressed) protobuf payload for sending.""" + return _HEADER.pack(0x00, len(payload)) + payload + + +def iter_frames(buf: bytes) -> Iterator[Tuple[int, bytes]]: + """Yield ``(flag, payload)`` for each frame in a grpc-web response body.""" + off, n = 0, len(buf) + while off + 5 <= n: + flag, length = _HEADER.unpack_from(buf, off) + off += 5 + if off + length > n: + raise ValueError("truncated grpc-web frame") + yield flag, buf[off : off + length] + off += length + if off != n: + raise ValueError("trailing bytes after final grpc-web frame") + + +def parse_trailers(raw: bytes) -> Dict[str, str]: + """Parse a trailer frame payload into a lower-cased header dict.""" + out: Dict[str, str] = {} + for line in raw.split(b"\r\n"): + if not line: + continue + key, _, value = line.partition(b":") + out[key.strip().decode("ascii").lower()] = value.strip().decode("ascii") + return out + + +def split_response(body: bytes) -> Tuple[List[bytes], Dict[str, str]]: + """Split a grpc-web response body into message payloads and trailers.""" + messages: List[bytes] = [] + trailers: Dict[str, str] = {} + for flag, payload in iter_frames(body): + if flag & _FLAG_TRAILER: + trailers.update(parse_trailers(payload)) + elif flag & _FLAG_COMPRESSED: + raise ValueError( + "compressed grpc-web message frames are not supported by this transport" + ) + else: + messages.append(payload) + return messages, trailers diff --git a/packages/grpc-web/src/weaviate_grpc_web/_sender.py b/packages/grpc-web/src/weaviate_grpc_web/_sender.py new file mode 100644 index 000000000..0ac0088e7 --- /dev/null +++ b/packages/grpc-web/src/weaviate_grpc_web/_sender.py @@ -0,0 +1,59 @@ +"""HTTP senders for the grpc-web transport. + +A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. +The default uses ``pyodide.http.pyfetch`` (browser fetch); a sender can be injected for +testing or for non-browser runtimes via :func:`weaviate_grpc_web.set_sender`. +""" + +from typing import Awaitable, Callable, Dict, Optional, Tuple + +Sender = Callable[ + [str, Dict[str, str], bytes, Optional[float]], + Awaitable[Tuple[int, Dict[str, str], bytes]], +] + + +async def pyfetch_sender( + url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] +) -> Tuple[int, Dict[str, str], bytes]: + """Default browser sender. + + Imports ``pyodide.http`` lazily so this module stays importable on CPython (where + ``pyodide`` does not exist). + """ + from pyodide.http import pyfetch # type: ignore[import-not-found] + + response = await pyfetch(url, method="POST", headers=headers, body=body) + data = await response.bytes() + try: + resp_headers = dict(response.headers) + except Exception: # pragma: no cover - header shape varies across Pyodide versions + resp_headers = {} + return int(response.status), resp_headers, data + + +def make_httpx_sender(client: Optional[object] = None) -> Sender: + """Build a sender backed by ``httpx.AsyncClient`` for CPython tests/integration. + + Targets a grpc-web transcoder (Envoy / connectrpc vanguard). + """ + import httpx + + async def _send( + url: str, headers: Dict[str, str], body: bytes, timeout: Optional[float] + ) -> Tuple[int, Dict[str, str], bytes]: + owns_client = client is None + active = client or httpx.AsyncClient() + assert isinstance(active, httpx.AsyncClient) + try: + response = await active.post(url, headers=headers, content=body, timeout=timeout) + return ( + response.status_code, + {k.lower(): v for k, v in response.headers.items()}, + response.content, + ) + finally: + if owns_client: + await active.aclose() + + return _send diff --git a/packages/grpc-web/src/weaviate_grpc_web/_shim.py b/packages/grpc-web/src/weaviate_grpc_web/_shim.py new file mode 100644 index 000000000..52b0ae635 --- /dev/null +++ b/packages/grpc-web/src/weaviate_grpc_web/_shim.py @@ -0,0 +1,265 @@ +"""A minimal pure-Python stand-in for the ``grpc`` API surface ``weaviate-client`` uses. + +It covers what ``weaviate-client`` touches at import time and on the async unary data +path. It is installed into ``sys.modules`` (as ``grpc``, ``grpc.aio``, ``grpc._utilities``, +``grpc.aio._typing``, ``grpc.experimental``) *before* ``import weaviate`` so the client +loads under Pyodide/Emscripten, where the real ``grpcio`` C-extension wheel does not +exist. The shim satisfies two contracts at once: + +1. **Import surface** — every ``import grpc`` / ``from grpc(.aio) import ...`` executed + while ``weaviate`` and its generated ``*_pb2_grpc`` stubs are imported + (``weaviate/config.py:4-5``, ``exceptions.py:7-8``, ``retry.py:5-6``, + ``connect/base.py:5-8``, ``connect/v4.py:24,29-32``, and the v6300 stub's + ``grpc.__version__`` / ``grpc._utilities.first_version_is_lower`` version gate). +2. **Runtime type contract** — :class:`AioChannel` becomes ``grpc.aio.Channel`` so the + real grpc-web channel (which subclasses it) passes the + ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py`` (lines 722, + 1241); :class:`AioRpcError` is the error the client catches and inspects via + ``.code()`` / ``.details()`` (``exceptions.py:62-76``, ``retry.py:30-31``). +""" + +import enum +import sys +import types +from typing import Any, Optional + +# grpcio reports 1.72.1 as the version that the v6300 generated stub requires; matching +# it makes the stub's import-time version gate pass. See weaviate/proto/v1/__init__.py. +FAKE_GRPC_VERSION = "1.72.1" + +_SHIM_MARKER = "__weaviate_grpc_web_shim__" + + +class StatusCode(enum.Enum): + """Mirror of ``grpc.StatusCode``. + + ``value`` is the canonical ``(int, str)`` tuple, matching grpcio so ``code.value[0]`` + / ``code.value[1]`` (``exceptions.py:63,66``) and ``code.name`` + (``connect/v4.py:1189``) behave identically. + """ + + OK = (0, "ok") + CANCELLED = (1, "cancelled") + UNKNOWN = (2, "unknown") + INVALID_ARGUMENT = (3, "invalid argument") + DEADLINE_EXCEEDED = (4, "deadline exceeded") + NOT_FOUND = (5, "not found") + ALREADY_EXISTS = (6, "already exists") + PERMISSION_DENIED = (7, "permission denied") + RESOURCE_EXHAUSTED = (8, "resource exhausted") + FAILED_PRECONDITION = (9, "failed precondition") + ABORTED = (10, "aborted") + OUT_OF_RANGE = (11, "out of range") + UNIMPLEMENTED = (12, "unimplemented") + INTERNAL = (13, "internal") + UNAVAILABLE = (14, "unavailable") + DATA_LOSS = (15, "data loss") + UNAUTHENTICATED = (16, "unauthenticated") + + +_BY_NUMBER = {member.value[0]: member for member in StatusCode} + + +def status_from_int(code: int) -> StatusCode: + """Map a numeric grpc-status to a :class:`StatusCode` (``UNKNOWN`` if unmapped).""" + return _BY_NUMBER.get(code, StatusCode.UNKNOWN) + + +class RpcError(Exception): + """Stand-in for ``grpc.RpcError`` (imported by ``retry.py``).""" + + +class Call: + """Stand-in for ``grpc.Call`` (imported by ``exceptions.py`` / ``retry.py``). + + Only used for ``isinstance``/type-import purposes; the async-only WASM path raises + :class:`AioRpcError`, never a sync ``Call``. + """ + + def code(self) -> StatusCode: # pragma: no cover - never instantiated under WASM + raise NotImplementedError + + def details(self) -> str: # pragma: no cover + raise NotImplementedError + + +class AioRpcError(RpcError): + """Stand-in for ``grpc.aio.AioRpcError``. + + Raised by the grpc-web multicallable on a non-OK status; exposes the same + ``code()`` / ``details()`` surface the client uses. + """ + + def __init__( + self, + code: StatusCode, + initial_metadata: Any = None, + trailing_metadata: Any = None, + details: str = "", + debug_error_string: Optional[str] = None, + ) -> None: + self._code = code + self._details = details + self._initial_metadata = initial_metadata + self._trailing_metadata = trailing_metadata + self._debug_error_string = debug_error_string + super().__init__(f"") + + def code(self) -> StatusCode: + return self._code + + def details(self) -> str: + return self._details + + def initial_metadata(self) -> Any: + return self._initial_metadata + + def trailing_metadata(self) -> Any: + return self._trailing_metadata + + def debug_error_string(self) -> Optional[str]: + return self._debug_error_string + + +class StreamStreamCall: + """Stand-in for ``grpc.aio.StreamStreamCall`` (imported as a type at ``v4.py:31``).""" + + +class ChannelCredentials: + """Stand-in for ``grpc.ChannelCredentials`` (imported by ``config.py:4``).""" + + +def ssl_channel_credentials(*_args: Any, **_kwargs: Any) -> ChannelCredentials: + return ChannelCredentials() + + +class SyncChannel: + """Stand-in for ``grpc.Channel`` (sync). + + Never instantiated under WASM — the sync channel factory raises (the WASM transport + is async-only). + """ + + +class AioChannel: + """Become ``grpc.aio.Channel``. + + The grpc-web channel subclasses this so the ``isinstance(..., grpc.aio.Channel)`` + assertions in ``connect/v4.py`` hold. + """ + + +def first_version_is_lower(_version: str, _other: str) -> bool: + """Stand-in for ``grpc._utilities.first_version_is_lower``. + + Returning ``False`` makes the v6300 stub's version gate + (``weaviate_pb2_grpc.py:17-29``) pass. + """ + return False + + +_ASYNC_ONLY_MESSAGE = ( + "weaviate-python-grpc-web provides an asynchronous-only gRPC transport under " + "WebAssembly/Pyodide. Use an async client (weaviate.use_async_with_local / " + "use_async_with_weaviate_cloud / use_async_with_custom, or WeaviateAsyncClient); " + "the synchronous client is not supported in the browser." +) + + +def _sync_channel_unsupported(*_args: Any, **_kwargs: Any) -> "AioChannel": + raise RuntimeError(_ASYNC_ONLY_MESSAGE) + + +def _aio_secure_channel( + target: Optional[str] = None, credentials: Any = None, options: Any = None, **_kw: Any +) -> AioChannel: + from ._channel import GrpcWebChannel + + return GrpcWebChannel(target=target, secure=True, options=options) + + +def _aio_insecure_channel( + target: Optional[str] = None, options: Any = None, **_kw: Any +) -> AioChannel: + from ._channel import GrpcWebChannel + + return GrpcWebChannel(target=target, secure=False, options=options) + + +def _noop(*_args: Any, **_kwargs: Any) -> None: + """Inert stand-in for imported-but-unused server-side stub-registration helpers. + + e.g. ``grpc.unary_unary_rpc_method_handler``: imported by generated ``*_pb2_grpc`` + code, never called by the client. + """ + return None + + +def is_installed() -> bool: + return getattr(sys.modules.get("grpc"), _SHIM_MARKER, False) is True + + +def install(force: bool = False) -> bool: + """Install the shim into ``sys.modules`` as ``grpc`` and submodules. + + On normal platforms this is a no-op unless ``force=True`` — we must never clobber a + real, working ``grpcio``. Under Emscripten the bootstrap calls this automatically. + Returns ``True`` if the shim is in place afterwards. + """ + if not force and sys.platform != "emscripten": + return False + if is_installed(): + return True + + # Modules are populated via __dict__.update — dynamic module synthesis, so static + # type checkers do not flag each attribute assignment. + utilities = types.ModuleType("grpc._utilities") + utilities.__dict__["first_version_is_lower"] = first_version_is_lower + + experimental = types.ModuleType("grpc.experimental") + experimental.__dict__.update(unary_unary=_noop, stream_stream=_noop) + + aio_typing = types.ModuleType("grpc.aio._typing") + aio_typing.__dict__["ChannelArgumentType"] = Any + + aio = types.ModuleType("grpc.aio") + aio.__dict__.update( + Channel=AioChannel, + AioRpcError=AioRpcError, + StreamStreamCall=StreamStreamCall, + secure_channel=_aio_secure_channel, + insecure_channel=_aio_insecure_channel, + _typing=aio_typing, + ) + + grpc_mod = types.ModuleType("grpc") + grpc_mod.__dict__.update( + { + "__version__": FAKE_GRPC_VERSION, + _SHIM_MARKER: True, + "StatusCode": StatusCode, + "RpcError": RpcError, + "Call": Call, + "Channel": SyncChannel, + "ChannelCredentials": ChannelCredentials, + "ssl_channel_credentials": ssl_channel_credentials, + "secure_channel": _sync_channel_unsupported, + "insecure_channel": _sync_channel_unsupported, + # Imported (never called) by generated *_pb2_grpc servicer/registration code. + "unary_unary_rpc_method_handler": _noop, + "stream_stream_rpc_method_handler": _noop, + "unary_stream_rpc_method_handler": _noop, + "stream_unary_rpc_method_handler": _noop, + "method_handlers_generic_handler": _noop, + "_utilities": utilities, + "experimental": experimental, + "aio": aio, + } + ) + + sys.modules["grpc"] = grpc_mod + sys.modules["grpc._utilities"] = utilities + sys.modules["grpc.experimental"] = experimental + sys.modules["grpc.aio"] = aio + sys.modules["grpc.aio._typing"] = aio_typing + return True diff --git a/packages/grpc-web/src/weaviate_grpc_web/py.typed b/packages/grpc-web/src/weaviate_grpc_web/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/grpc-web/tests/conftest.py b/packages/grpc-web/tests/conftest.py new file mode 100644 index 000000000..fe4afb09d --- /dev/null +++ b/packages/grpc-web/tests/conftest.py @@ -0,0 +1,7 @@ +import pathlib +import sys + +# Make the package importable without an editable install. +_SRC = pathlib.Path(__file__).resolve().parents[1] / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) diff --git a/packages/grpc-web/tests/test_framing.py b/packages/grpc-web/tests/test_framing.py new file mode 100644 index 000000000..320aff7f6 --- /dev/null +++ b/packages/grpc-web/tests/test_framing.py @@ -0,0 +1,59 @@ +import struct + +import pytest + +from weaviate_grpc_web._framing import ( + encode_message, + iter_frames, + parse_trailers, + split_response, +) + + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload + + +def test_encode_message_round_trip(): + framed = encode_message(b"hello") + frames = list(iter_frames(framed)) + assert frames == [(0x00, b"hello")] + + +def test_split_response_message_and_trailer(): + body = _frame(b"payload") + _frame(b"grpc-status:0\r\ngrpc-message:\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [b"payload"] + assert trailers["grpc-status"] == "0" + assert trailers["grpc-message"] == "" + + +def test_split_response_multiple_messages(): + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [b"a", b"bb"] + assert trailers["grpc-status"] == "0" + + +def test_split_response_trailers_only(): + body = _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80) + messages, trailers = split_response(body) + assert messages == [] + assert trailers == {"grpc-status": "7", "grpc-message": "denied"} + + +def test_parse_trailers_lowercases_keys(): + parsed = parse_trailers(b"Grpc-Status:0\r\nGrpc-Message:ok\r\n") + assert parsed == {"grpc-status": "0", "grpc-message": "ok"} + + +def test_truncated_frame_raises(): + framed = encode_message(b"hello")[:-2] + with pytest.raises(ValueError): + list(iter_frames(framed)) + + +def test_compressed_message_frame_rejected(): + body = _frame(b"x", 0x01) + with pytest.raises(ValueError): + split_response(body) diff --git a/packages/grpc-web/tests/test_shim_install.py b/packages/grpc-web/tests/test_shim_install.py new file mode 100644 index 000000000..c950da902 --- /dev/null +++ b/packages/grpc-web/tests/test_shim_install.py @@ -0,0 +1,111 @@ +"""Shim/import tests. + +Installing the shim replaces ``sys.modules['grpc']`` process-wide, so each scenario runs +in a fresh subprocess to avoid clobbering the real ``grpc`` used by the rest of the suite. +""" + +import pathlib +import subprocess +import sys +import textwrap + +_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") + + +def _run(body: str) -> subprocess.CompletedProcess: + script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + textwrap.dedent(body) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + +def test_import_weaviate_under_shim(): + result = _run( + """ + import weaviate_grpc_web + assert weaviate_grpc_web.install(force=True) is True + assert weaviate_grpc_web.is_installed() + + import grpc + assert getattr(grpc, "__weaviate_grpc_web_shim__", False) is True + assert grpc.__version__ == "1.72.1" + assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False + from grpc.aio._typing import ChannelArgumentType # noqa: F401 + + import weaviate # must not raise even though grpcio is shimmed + from weaviate.proto.v1 import weaviate_pb2_grpc + from weaviate_grpc_web import GrpcWebChannel + + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + assert stub.Search is not None + assert stub.BatchObjects is not None + assert stub.BatchDelete is not None + assert isinstance(ch, grpc.aio.Channel) + print("OK") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_sync_channel_factory_raises_async_only(): + result = _run( + """ + import weaviate_grpc_web + weaviate_grpc_web.install(force=True) + import grpc + try: + grpc.insecure_channel("localhost:50051") + except RuntimeError as exc: + assert "async" in str(exc).lower() + print("OK") + else: + raise AssertionError("expected sync channel factory to raise") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_real_proto_unary_round_trip_under_shim(): + result = _run( + """ + import asyncio + import struct + import weaviate_grpc_web + weaviate_grpc_web.install(force=True) + + import weaviate # noqa: F401 + from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc + + reply = tenants_pb2.TenantsGetReply() + payload = reply.SerializeToString() + + def frame(p, flag=0x00): + return struct.pack(">BI", flag, len(p)) + p + + body = frame(payload) + frame(b"grpc-status:0\\r\\n", 0x80) + + async def sender(url, headers, body_in, timeout): + assert headers["authorization"] == "Bearer k" + assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") + return 200, {}, body + + weaviate_grpc_web.set_sender(sender) + from weaviate_grpc_web import GrpcWebChannel + ch = GrpcWebChannel("localhost:50051", secure=False) + stub = weaviate_pb2_grpc.WeaviateStub(ch) + + async def main(): + res = await stub.TenantsGet( + tenants_pb2.TenantsGetRequest(), + metadata=[("authorization", "Bearer k")], + timeout=5, + ) + assert isinstance(res, tenants_pb2.TenantsGetReply) + print("OK") + + asyncio.run(main()) + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/packages/grpc-web/tests/test_transport.py b/packages/grpc-web/tests/test_transport.py new file mode 100644 index 000000000..c137c36b4 --- /dev/null +++ b/packages/grpc-web/tests/test_transport.py @@ -0,0 +1,158 @@ +"""In-process tests for the grpc-web channel/multicallable. + +These exercise the transport classes directly (they import their grpc base classes from +``weaviate_grpc_web._shim``, not from ``sys.modules['grpc']``), so no shim install is +needed and the real ``grpc`` in the dev environment is left untouched. +""" + +import asyncio +import struct +from typing import Dict, List, Optional, Tuple + +import pytest + +from weaviate_grpc_web._channel import GrpcWebChannel, set_sender +from weaviate_grpc_web._shim import AioChannel, AioRpcError, StatusCode + + +def _frame(payload: bytes, flag: int = 0x00) -> bytes: + return struct.pack(">BI", flag, len(payload)) + payload + + +def _ok_response(payload: bytes) -> bytes: + return _frame(payload) + _frame(b"grpc-status:0\r\n", 0x80) + + +class FakeSender: + def __init__( + self, status: int = 200, headers: Optional[Dict[str, str]] = None, body: bytes = b"" + ): + self.status = status + self.headers = headers or {} + self.body = body + self.calls: List[Tuple[str, Dict[str, str], bytes, Optional[float]]] = [] + + async def __call__(self, url, headers, body, timeout): + self.calls.append((url, headers, body, timeout)) + return self.status, self.headers, self.body + + +def _channel(sender: FakeSender, secure: bool = False) -> GrpcWebChannel: + return GrpcWebChannel("example.com:443", secure=secure, sender=sender) + + +def test_grpcwebchannel_is_grpc_aio_channel(): + assert issubclass(GrpcWebChannel, AioChannel) + assert isinstance(_channel(FakeSender()), AioChannel) + + +def test_unary_success_round_trip(): + sender = FakeSender(body=_ok_response(b"reply-bytes")) + channel = _channel(sender) + mc = channel.unary_unary( + "/weaviate.v1.Weaviate/Search", + request_serializer=lambda x: x, + response_deserializer=lambda b: b, + _registered_method=True, + ) + + result = asyncio.run(mc(b"request-bytes", metadata=[("authorization", "Bearer k")], timeout=5)) + + assert result == b"reply-bytes" + url, headers, body, timeout = sender.calls[0] + assert url == "http://example.com:443/weaviate.v1.Weaviate/Search" + assert body == _frame(b"request-bytes") + assert headers["content-type"] == "application/grpc-web+proto" + assert headers["authorization"] == "Bearer k" + assert headers["grpc-timeout"] == "5000m" + assert timeout == 5 + + +def test_secure_channel_uses_https(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender, secure=True) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q")) + assert sender.calls[0][0].startswith("https://example.com:443/") + + +def test_health_call_without_metadata(): + sender = FakeSender(body=_ok_response(b"pong")) + channel = _channel(sender) + mc = channel.unary_unary("/grpc.health.v1.Health/Check", lambda x: x, lambda b: b) + # mirrors connect/v4.py:316 — request + timeout, no metadata + assert asyncio.run(mc(b"ping", timeout=2)) == b"pong" + + +def test_error_trailer_raises_aiorpcerror(): + body = _frame(b"grpc-status:7\r\ngrpc-message:nope\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.PERMISSION_DENIED + assert excinfo.value.code().name == "PERMISSION_DENIED" + assert excinfo.value.details() == "nope" + + +def test_percent_encoded_grpc_message_decoded(): + body = _frame(b"grpc-status:5\r\ngrpc-message:not%20found\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.details() == "not found" + + +def test_trailers_only_status_in_http_headers(): + channel = _channel( + FakeSender(status=200, headers={"grpc-status": "16", "grpc-message": "auth"}, body=b"") + ) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.UNAUTHENTICATED + + +def test_http_error_without_grpc_status_maps_to_code(): + channel = _channel(FakeSender(status=403, headers={}, body=b"")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.PERMISSION_DENIED + + +def test_binary_metadata_base64_encoded(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q", metadata=[("trace-bin", b"\x00\x01\x02")])) + assert sender.calls[0][1]["trace-bin"] == "AAEC" + + +def test_stream_stream_raises_clear_error(): + channel = _channel(FakeSender()) + mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) + with pytest.raises(RuntimeError) as excinfo: + mc(request_iterator=iter([]), timeout=5, metadata=None) + assert "not supported over grpc-web" in str(excinfo.value) + + +def test_close_is_awaitable_noop(): + channel = _channel(FakeSender()) + assert asyncio.run(channel.close()) is None + + +def test_set_sender_overrides_default(): + sender = FakeSender(body=_ok_response(b"y")) + set_sender(sender) + try: + channel = GrpcWebChannel("h:1", secure=False) # no explicit sender + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert asyncio.run(mc(b"q")) == b"y" + finally: + # restore the real default so other tests/processes are unaffected + from weaviate_grpc_web._sender import pyfetch_sender + + set_sender(pyfetch_sender) diff --git a/proto_test/test_proto.py b/proto_test/test_proto.py index bedbf10c3..a964393ec 100644 --- a/proto_test/test_proto.py +++ b/proto_test/test_proto.py @@ -1,5 +1,7 @@ +import importlib +from importlib.metadata import PackageNotFoundError, version as metadata_version + import pytest -from importlib.metadata import version as metadata_version from packaging import version @@ -17,3 +19,30 @@ def test_proto_import(): import weaviate assert weaviate.version is not None + + +def test_grpcio_metadata_fallback_under_emscripten(monkeypatch): + """Fall back for grpcio when its metadata is absent; protobuf still surfaces. + + Under Pyodide/Emscripten grpcio is excluded via an environment marker, so its + distribution metadata is missing and ``get_version`` must fall back to a working + proto variant; a genuinely missing protobuf is still surfaced, not masked. + """ + mod = importlib.import_module("weaviate.proto.v1") + + def raises(pkg: str) -> str: + raise PackageNotFoundError(pkg) + + monkeypatch.setattr(mod, "metadata_version", raises) + + assert str(mod.get_version("grpcio")) == "1.72.1" + with pytest.raises(PackageNotFoundError): + mod.get_version("protobuf") + + +def test_get_version_passthrough_when_installed(monkeypatch): + """On a normal install the real version is returned unchanged (no fallback).""" + mod = importlib.import_module("weaviate.proto.v1") + monkeypatch.setattr(mod, "metadata_version", lambda pkg: "1.2.3") + assert str(mod.get_version("grpcio")) == "1.2.3" + assert str(mod.get_version("protobuf")) == "1.2.3" diff --git a/setup.cfg b/setup.cfg index 0b5ba855a..7343116a5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ install_requires = # When bumping authlib to >=2.0.0, remove the `authlib.jose` deprecation # warning filter implemented in `weaviate/_authlib_compat.py`. pydantic>=2.12.0,<3.0.0 - grpcio>=1.59.5,<1.80.0 + grpcio>=1.59.5,<1.80.0; sys_platform != "emscripten" protobuf>=4.21.6,<7.0.0 packaging>=21.0 python_requires = >=3.10 diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index 09171e683..a3821fed5 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -11,12 +11,26 @@ from packaging import version -from importlib.metadata import version as metadata_version +from importlib.metadata import PackageNotFoundError, version as metadata_version from weaviate.exceptions import WeaviateProtobufIncompatibility -def get_version(pkg: str)-> version.Version: - return version.parse(metadata_version(pkg)) +# Fallback grpcio version used only when grpcio is not installed as a distribution. +# This happens under Pyodide/Emscripten, where grpcio has no wheel and is excluded +# via the `sys_platform != "emscripten"` marker in setup.cfg; the grpc module itself +# is provided there by a pure-Python shim (see the weaviate-python-grpc-web package). +# On every normal install grpcio's metadata is present and the real version is used, +# so this branch is not taken. Restricted to grpcio so that a genuinely missing +# protobuf (which is required and pure-Python under Pyodide) is never masked. +_GRPCIO_FALLBACK_VERSION = "1.72.1" + +def get_version(pkg: str) -> version.Version: + try: + return version.parse(metadata_version(pkg)) + except PackageNotFoundError: + if pkg == "grpcio": + return version.parse(_GRPCIO_FALLBACK_VERSION) + raise pb_version, grpc_version = get_version("protobuf"), get_version("grpcio") if pb_version >= version.parse("6.30.0"): From 99f5356e80823562f032ce56be258efd9f0d8d67 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:30:53 +0300 Subject: [PATCH 02/27] feat(grpc-web): support grpc-web on the REST host:port via a base-path prefix Add a configurable gRPC base-path prefix so the client can talk to a grpc-web endpoint multiplexed onto the REST host:port (the production wire contract: grpc-web served under "/grpc-web/" via an in-process transcoder). - ConnectionParams gains `grpc_path_prefix`, threaded through `from_params` / `from_url` and the `connect_to_custom` / `use_async_with_custom` helpers. Normalized to a single leading slash, no trailing slash; None/"" == native gRPC. - `_check_port_collision` no longer rejects the same host:port when a grpc-web prefix is set; native gRPC (no prefix) still raises, unchanged. - `_grpc_channel` forwards the prefix to the transport as a ("grpc-web.path_prefix", prefix) channel option only in grpc-web mode, so the native channel options stay byte-for-byte unchanged. - weaviate-python-grpc-web: the shim's channel factories read that option and GrpcWebChannel prepends the prefix, so requests go to ://:/weaviate.v1.Weaviate/. Tested: new ConnectionParams unit tests (collision relaxed only with a prefix; native same-port still raises; option forwarded/omitted) and grpc-web transport tests for the prefixed/normalized URL. End-to-end verified against a vanguard transcoder on a shared host:port (insert_many/fetch_objects/aggregate over grpc-web with grpc_host == http_host == localhost:8090). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/weaviate_grpc_web/_channel.py | 6 +- .../grpc-web/src/weaviate_grpc_web/_shim.py | 22 ++- packages/grpc-web/tests/test_transport.py | 42 ++++++ test/test_connection_params.py | 128 ++++++++++++++++++ weaviate/connect/base.py | 36 ++++- weaviate/connect/helpers.py | 14 ++ 6 files changed, 243 insertions(+), 5 deletions(-) create mode 100644 test/test_connection_params.py diff --git a/packages/grpc-web/src/weaviate_grpc_web/_channel.py b/packages/grpc-web/src/weaviate_grpc_web/_channel.py index 5e7b9bd55..2402bec02 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_channel.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_channel.py @@ -125,12 +125,16 @@ def __init__( target: Optional[str], secure: bool, options: Any = None, + path_prefix: str = "", sender: Optional[Sender] = None, ) -> None: if not target: raise ValueError("GrpcWebChannel requires a target (host:port)") scheme = "https" if secure else "http" self._base_url = f"{scheme}://{target}" + # Normalize to a single leading slash and no trailing slash; "" == native path. + cleaned = (path_prefix or "").strip("/") + self._path_prefix = f"/{cleaned}" if cleaned else "" self._sender: Sender = sender or get_sender() def unary_unary( @@ -173,7 +177,7 @@ async def _unary( if timeout is not None: headers["grpc-timeout"] = _encode_timeout(timeout) - url = self._base_url + path + url = self._base_url + self._path_prefix + path status, resp_headers, body = await self._sender( url, headers, encode_message(payload), timeout ) diff --git a/packages/grpc-web/src/weaviate_grpc_web/_shim.py b/packages/grpc-web/src/weaviate_grpc_web/_shim.py index 52b0ae635..c8226cecc 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_shim.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_shim.py @@ -170,12 +170,25 @@ def _sync_channel_unsupported(*_args: Any, **_kwargs: Any) -> "AioChannel": raise RuntimeError(_ASYNC_ONLY_MESSAGE) +def _path_prefix_from_options(options: Any) -> str: + """Extract the ``("grpc-web.path_prefix", prefix)`` channel option, or "" if absent.""" + for item in options or (): + if isinstance(item, (tuple, list)) and len(item) == 2 and item[0] == "grpc-web.path_prefix": + return item[1] or "" + return "" + + def _aio_secure_channel( target: Optional[str] = None, credentials: Any = None, options: Any = None, **_kw: Any ) -> AioChannel: from ._channel import GrpcWebChannel - return GrpcWebChannel(target=target, secure=True, options=options) + return GrpcWebChannel( + target=target, + secure=True, + options=options, + path_prefix=_path_prefix_from_options(options), + ) def _aio_insecure_channel( @@ -183,7 +196,12 @@ def _aio_insecure_channel( ) -> AioChannel: from ._channel import GrpcWebChannel - return GrpcWebChannel(target=target, secure=False, options=options) + return GrpcWebChannel( + target=target, + secure=False, + options=options, + path_prefix=_path_prefix_from_options(options), + ) def _noop(*_args: Any, **_kwargs: Any) -> None: diff --git a/packages/grpc-web/tests/test_transport.py b/packages/grpc-web/tests/test_transport.py index c137c36b4..33a8eafbf 100644 --- a/packages/grpc-web/tests/test_transport.py +++ b/packages/grpc-web/tests/test_transport.py @@ -144,6 +144,48 @@ def test_close_is_awaitable_noop(): assert asyncio.run(channel.close()) is None +def test_path_prefix_prepended_to_url(): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel( + "example.com:8090", secure=False, sender=sender, path_prefix="/grpc-web" + ) + mc = channel.unary_unary("/weaviate.v1.Weaviate/Search", lambda x: x, lambda b: b) + asyncio.run(mc(b"q")) + assert sender.calls[0][0] == "http://example.com:8090/grpc-web/weaviate.v1.Weaviate/Search" + + +@pytest.mark.parametrize( + "raw,expected_url", + [ + ("grpc-web", "http://h:1/grpc-web/svc/M"), + ("/grpc-web/", "http://h:1/grpc-web/svc/M"), + ("/a/b", "http://h:1/a/b/svc/M"), + ("", "http://h:1/svc/M"), + ], +) +def test_path_prefix_normalized_in_url(raw, expected_url): + sender = FakeSender(body=_ok_response(b"r")) + channel = GrpcWebChannel("h:1", secure=False, sender=sender, path_prefix=raw) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q")) + assert sender.calls[0][0] == expected_url + + +def test_shim_factory_extracts_path_prefix_option(): + from weaviate_grpc_web._shim import _aio_insecure_channel + + with_prefix = _aio_insecure_channel( + target="h:1", + options=[("grpc.max_send_message_length", 1), ("grpc-web.path_prefix", "/grpc-web")], + ) + assert with_prefix._path_prefix == "/grpc-web" + + without_prefix = _aio_insecure_channel( + target="h:1", options=[("grpc.max_send_message_length", 1)] + ) + assert without_prefix._path_prefix == "" + + def test_set_sender_overrides_default(): sender = FakeSender(body=_ok_response(b"y")) set_sender(sender) diff --git a/test/test_connection_params.py b/test/test_connection_params.py new file mode 100644 index 000000000..69b07f5a4 --- /dev/null +++ b/test/test_connection_params.py @@ -0,0 +1,128 @@ +import pytest +from pydantic import ValidationError + +import weaviate.connect.base as base_mod +from weaviate.connect.base import ConnectionParams + + +def test_same_host_port_raises_without_prefix() -> None: + with pytest.raises(ValidationError, match="must be different"): + ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + ) + + +def test_from_url_same_host_port_raises_without_prefix() -> None: + with pytest.raises(ValidationError, match="must be different"): + ConnectionParams.from_url("http://localhost:8090", grpc_port=8090) + + +def test_same_host_port_allowed_with_grpc_web_prefix() -> None: + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + assert params._grpc_web_path_prefix == "/grpc-web" + + +def test_from_url_same_host_port_allowed_with_prefix() -> None: + params = ConnectionParams.from_url( + "http://localhost:8090", grpc_port=8090, grpc_path_prefix="/grpc-web" + ) + assert params._grpc_web_path_prefix == "/grpc-web" + + +def test_different_ports_still_ok_without_prefix() -> None: + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=50051, + grpc_secure=False, + ) + assert params._grpc_web_path_prefix == "" + + +@pytest.mark.parametrize( + "raw,expected", + [ + (None, ""), + ("", ""), + ("/", ""), + ("grpc-web", "/grpc-web"), + ("/grpc-web", "/grpc-web"), + ("grpc-web/", "/grpc-web"), + ("/a/b/", "/a/b"), + ], +) +def test_path_prefix_normalization(raw, expected) -> None: + params = ConnectionParams.from_params( + http_host="h", + http_port=8080, + http_secure=False, + grpc_host="g", + grpc_port=50051, + grpc_secure=False, + grpc_path_prefix=raw, + ) + assert params._grpc_web_path_prefix == expected + + +def test_grpc_channel_forwards_path_prefix_option(monkeypatch) -> None: + captured: dict = {} + + def fake_insecure_channel(target, options=None, **kwargs): + captured["target"] = target + captured["options"] = options + return "CHANNEL" + + monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) + + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + channel = params._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) + + assert channel == "CHANNEL" + assert captured["target"] == "localhost:8090" + assert ("grpc-web.path_prefix", "/grpc-web") in captured["options"] + + +def test_grpc_channel_omits_option_without_prefix(monkeypatch) -> None: + captured: dict = {} + + def fake_insecure_channel(target, options=None, **kwargs): + captured["options"] = options + return "CHANNEL" + + monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) + + params = ConnectionParams.from_params( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=50051, + grpc_secure=False, + ) + params._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) + + option_keys = [key for key, _ in captured["options"]] + assert "grpc-web.path_prefix" not in option_keys diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index 99607e3ae..df2c79a28 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -47,9 +47,19 @@ def is_gcp(self) -> bool: class ConnectionParams(BaseModel): http: ProtocolParams grpc: ProtocolParams + # Optional base-path prefix for a grpc-web endpoint served on the REST host:port + # (e.g. "/grpc-web"). None/"" means native gRPC. When set, sharing the REST + # host:port is permitted and the prefix is forwarded to the grpc-web transport. + grpc_path_prefix: Optional[str] = None @classmethod - def from_url(cls, url: str, grpc_port: int, grpc_secure: bool = False) -> "ConnectionParams": + def from_url( + cls, + url: str, + grpc_port: int, + grpc_secure: bool = False, + grpc_path_prefix: Optional[str] = None, + ) -> "ConnectionParams": parsed_url = urlparse(url) if parsed_url.scheme not in ["http", "https"]: raise ValueError(f"Unsupported scheme: {parsed_url.scheme}") @@ -69,6 +79,7 @@ def from_url(cls, url: str, grpc_port: int, grpc_secure: bool = False) -> "Conne port=grpc_port, secure=grpc_secure or parsed_url.scheme == "https", ), + grpc_path_prefix=grpc_path_prefix, ) @classmethod @@ -80,6 +91,7 @@ def from_params( grpc_host: str, grpc_port: int, grpc_secure: bool, + grpc_path_prefix: Optional[str] = None, ) -> "ConnectionParams": return cls( http=ProtocolParams( @@ -92,6 +104,7 @@ def from_params( port=grpc_port, secure=grpc_secure, ), + grpc_path_prefix=grpc_path_prefix, ) def is_gcp_on_wcd(self) -> bool: @@ -99,7 +112,10 @@ def is_gcp_on_wcd(self) -> bool: @model_validator(mode="after") def _check_port_collision(self: T) -> T: - if self.http.host == self.grpc.host and self.http.port == self.grpc.port: + same_endpoint = self.http.host == self.grpc.host and self.http.port == self.grpc.port + # grpc-web can be multiplexed onto the REST port under a base-path prefix, so a + # shared host:port is only a conflict for native gRPC (no prefix configured). + if same_endpoint and self._grpc_web_path_prefix == "": raise ValueError("http.port and grpc.port must be different if using the same host") return self @@ -111,6 +127,16 @@ def _grpc_address(self) -> Tuple[str, int]: def _grpc_target(self) -> str: return f"{self.grpc.host}:{self.grpc.port}" + @property + def _grpc_web_path_prefix(self) -> str: + """Return the normalized grpc-web base-path prefix; "" means native gRPC. + + A configured prefix is returned with a single leading slash and no trailing + slash (e.g. "grpc-web/" -> "/grpc-web"); empty/None -> "" (native gRPC). + """ + cleaned = (self.grpc_path_prefix or "").strip("/") + return f"/{cleaned}" if cleaned else "" + def _grpc_channel( self, proxies: Dict[str, str], @@ -134,6 +160,12 @@ def _grpc_channel( if grpc_config is not None and grpc_config.channel_options is not None: options.extend(grpc_config.channel_options) + # In grpc-web mode, forward the base-path prefix to the transport via channel + # options (consumed by the weaviate-python-grpc-web shim). Not added for native + # gRPC, so the native channel options stay byte-for-byte unchanged. + if (prefix := self._grpc_web_path_prefix) != "": + options.append(("grpc-web.path_prefix", prefix)) + if is_async: mod = grpc.aio else: diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index 29faaa3c2..c9aed194b 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -290,6 +290,7 @@ def connect_to_custom( additional_config: Optional[AdditionalConfig] = None, auth_credentials: Optional[AuthCredentials] = None, skip_init_checks: bool = False, + grpc_path_prefix: Optional[str] = None, ) -> WeaviateClient: """Connect to a Weaviate instance with custom connection parameters. @@ -312,6 +313,11 @@ def connect_to_custom( a bearer token, in which case use `weaviate.classes.init.Auth.bearer_token()`, a client secret, in which case use `weaviate.classes.init.Auth.client_credentials()` or a username and password, in which case use `weaviate.classes.init.Auth.client_password()`. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. + grpc_path_prefix: Optional base-path prefix for a grpc-web endpoint served on the + same host:port as REST (e.g. "/grpc-web"). When set, gRPC requests are sent + over grpc-web to ``://:/...`` and sharing + the REST host:port is allowed. Requires the ``weaviate-python-grpc-web`` + package. Defaults to None (native gRPC). Returns: The client connected to the instance with the required parameters set appropriately. @@ -353,6 +359,7 @@ def connect_to_custom( grpc_host=grpc_host, grpc_port=grpc_port, grpc_secure=grpc_secure, + grpc_path_prefix=grpc_path_prefix, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, @@ -587,6 +594,7 @@ def use_async_with_custom( additional_config: Optional[AdditionalConfig] = None, auth_credentials: Optional[AuthCredentials] = None, skip_init_checks: bool = False, + grpc_path_prefix: Optional[str] = None, ) -> WeaviateAsyncClient: """Create an async client object ready to connect to a Weaviate instance with custom connection parameters. @@ -609,6 +617,11 @@ def use_async_with_custom( a bearer token, in which case use `weaviate.classes.init.Auth.bearer_token()`, a client secret, in which case use `weaviate.classes.init.Auth.client_credentials()` or a username and password, in which case use `weaviate.classes.init.Auth.client_password()`. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. + grpc_path_prefix: Optional base-path prefix for a grpc-web endpoint served on the + same host:port as REST (e.g. "/grpc-web"). When set, gRPC requests are sent + over grpc-web to ``://:/...`` and sharing + the REST host:port is allowed. Requires the ``weaviate-python-grpc-web`` + package. Defaults to None (native gRPC). Returns: The client connected to the instance with the required parameters set appropriately. @@ -652,6 +665,7 @@ def use_async_with_custom( grpc_host=grpc_host, grpc_port=grpc_port, grpc_secure=grpc_secure, + grpc_path_prefix=grpc_path_prefix, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, From fe208c09a8e899b136323f84404d6c9fd6790489 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:51:38 +0300 Subject: [PATCH 03/27] fix(grpc-web): surface transport/parse failures as AioRpcError, enforce client deadline Addresses Copilot review feedback on #2056. The grpc-web transport boundary now only ever raises grpc.aio.AioRpcError, and call timeouts are enforced client-side: - GrpcWebChannel._unary wraps the send in asyncio.wait_for(timeout) so a stalled browser request can't hang forever (pyfetch has no timeout argument of its own); a timeout maps to AioRpcError(DEADLINE_EXCEEDED). - Transport/network errors raised by the sender map to AioRpcError(UNAVAILABLE), which the client's existing exponential backoff retries. - Malformed/truncated/compressed framing and a non-integer grpc-status (which previously escaped as a bare ValueError) map to AioRpcError(INTERNAL). New transport tests cover each path (deadline, unavailable, malformed frame, malformed grpc-status). Existing transport/framing/shim tests and the same-host:port end-to-end check against a vanguard transcoder still pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/weaviate_grpc_web/_channel.py | 39 ++++++++++++++++-- .../grpc-web/src/weaviate_grpc_web/_sender.py | 3 +- packages/grpc-web/tests/test_transport.py | 41 +++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/packages/grpc-web/src/weaviate_grpc_web/_channel.py b/packages/grpc-web/src/weaviate_grpc_web/_channel.py index 2402bec02..853e4fc0b 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_channel.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_channel.py @@ -12,6 +12,7 @@ raises a clear error. """ +import asyncio import base64 import urllib.parse from typing import Any, Callable, Dict, Optional @@ -178,10 +179,40 @@ async def _unary( headers["grpc-timeout"] = _encode_timeout(timeout) url = self._base_url + self._path_prefix + path - status, resp_headers, body = await self._sender( - url, headers, encode_message(payload), timeout - ) - return self._handle_response(status, resp_headers, body, deserialize) + framed = encode_message(payload) + + # Send. Enforce a client-side deadline (the grpc-timeout header is server-side + # only; pyfetch ignores its timeout arg, so without this a stalled request could + # hang forever). Any transport/parse failure is surfaced as AioRpcError so the + # client only ever sees gRPC error types (never a bare ValueError/TimeoutError). + try: + send = self._sender(url, headers, framed, timeout) + if timeout is not None: + status, resp_headers, body = await asyncio.wait_for(send, timeout) + else: + status, resp_headers, body = await send + except AioRpcError: + raise + except asyncio.TimeoutError as exc: + raise AioRpcError( + code=StatusCode.DEADLINE_EXCEEDED, + details=f"grpc-web request to {path} timed out after {timeout}s", + ) from exc + except Exception as exc: # network/transport failure -> retryable UNAVAILABLE + raise AioRpcError( + code=StatusCode.UNAVAILABLE, + details=f"grpc-web transport error for {path}: {exc}", + ) from exc + + try: + return self._handle_response(status, resp_headers, body, deserialize) + except AioRpcError: + raise + except Exception as exc: # malformed framing / status / payload + raise AioRpcError( + code=StatusCode.INTERNAL, + details=f"malformed grpc-web response for {path}: {exc}", + ) from exc @staticmethod def _handle_response( diff --git a/packages/grpc-web/src/weaviate_grpc_web/_sender.py b/packages/grpc-web/src/weaviate_grpc_web/_sender.py index 0ac0088e7..2a879ec49 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_sender.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_sender.py @@ -19,7 +19,8 @@ async def pyfetch_sender( """Default browser sender. Imports ``pyodide.http`` lazily so this module stays importable on CPython (where - ``pyodide`` does not exist). + ``pyodide`` does not exist). ``pyfetch`` has no timeout parameter of its own; the + call deadline is enforced by ``GrpcWebChannel._unary`` via ``asyncio.wait_for``. """ from pyodide.http import pyfetch # type: ignore[import-not-found] diff --git a/packages/grpc-web/tests/test_transport.py b/packages/grpc-web/tests/test_transport.py index 33a8eafbf..753f227de 100644 --- a/packages/grpc-web/tests/test_transport.py +++ b/packages/grpc-web/tests/test_transport.py @@ -139,6 +139,47 @@ def test_stream_stream_raises_clear_error(): assert "not supported over grpc-web" in str(excinfo.value) +def test_timeout_maps_to_deadline_exceeded(): + async def slow_sender(url, headers, body, timeout): + await asyncio.sleep(0.5) + return 200, {}, _ok_response(b"x") + + channel = GrpcWebChannel("h:1", secure=False, sender=slow_sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q", timeout=0.01)) + assert excinfo.value.code() is StatusCode.DEADLINE_EXCEEDED + + +def test_transport_exception_maps_to_unavailable(): + async def boom(url, headers, body, timeout): + raise ConnectionError("connection refused") + + channel = GrpcWebChannel("h:1", secure=False, sender=boom) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.UNAVAILABLE + + +def test_malformed_frame_maps_to_internal(): + # A 3-byte body cannot contain even a 5-byte frame header -> framing ValueError. + channel = _channel(FakeSender(body=b"\x00\x00\x00")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + + +def test_malformed_grpc_status_maps_to_internal(): + body = _frame(b"grpc-status:notanint\r\n", 0x80) + channel = _channel(FakeSender(body=body)) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + + def test_close_is_awaitable_noop(): channel = _channel(FakeSender()) assert asyncio.run(channel.close()) is None From 833bb59b6174f40a61a9080d92048ad12c006726 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:58:03 +0300 Subject: [PATCH 04/27] test(grpc-web): skip get_version unit tests when grpcio/protobuf are incompatible The proto version-gate CI matrix installs deliberately-incompatible grpcio/protobuf pairs so importing weaviate.proto.v1 raises WeaviateProtobufIncompatibility (by design, covered by test_proto_import). The two get_version fallback tests added in this PR import weaviate.proto.v1, which re-runs that gate and errors in those cells. Skip them when the installed pair is incompatible (computed without importing weaviate); the fallback is still exercised in every compatible cell. Co-Authored-By: Claude Opus 4.8 (1M context) --- proto_test/test_proto.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/proto_test/test_proto.py b/proto_test/test_proto.py index a964393ec..7b089d875 100644 --- a/proto_test/test_proto.py +++ b/proto_test/test_proto.py @@ -5,6 +5,26 @@ from packaging import version +# The CI matrix deliberately installs incompatible grpcio/protobuf pairs to exercise the +# version gate in weaviate/proto/v1/__init__.py. In those cells the package raises on +# import (covered by test_proto_import), so the get_version unit tests below are skipped; +# the fallback they test still runs in every compatible cell. This check imports nothing +# from weaviate, so the test module always loads. +def _versions_incompatible() -> bool: + """Whether the installed grpcio/protobuf pair makes ``import weaviate.proto.v1`` raise.""" + try: + grpc_ver = version.parse(metadata_version("grpcio")) + pb_ver = version.parse(metadata_version("protobuf")) + except PackageNotFoundError: + return False + return (pb_ver >= version.parse("6.30.0") and grpc_ver < version.parse("1.72.0")) or ( + pb_ver >= version.parse("5.26.1") and grpc_ver < version.parse("1.63.0") + ) + + +_INCOMPATIBLE_GRPC_PB = _versions_incompatible() + + def test_proto_import(): grpc_ver = version.parse(metadata_version("grpcio")) pb_ver = version.parse(metadata_version("protobuf")) @@ -21,6 +41,12 @@ def test_proto_import(): assert weaviate.version is not None +@pytest.mark.skipif( + _INCOMPATIBLE_GRPC_PB, + reason="weaviate.proto.v1 cannot be imported with an incompatible grpcio/protobuf " + "pair (CI version-gate matrix); the gate is covered by test_proto_import and the " + "fallback is exercised in every compatible cell", +) def test_grpcio_metadata_fallback_under_emscripten(monkeypatch): """Fall back for grpcio when its metadata is absent; protobuf still surfaces. @@ -40,6 +66,12 @@ def raises(pkg: str) -> str: mod.get_version("protobuf") +@pytest.mark.skipif( + _INCOMPATIBLE_GRPC_PB, + reason="weaviate.proto.v1 cannot be imported with an incompatible grpcio/protobuf " + "pair (CI version-gate matrix); the gate is covered by test_proto_import and the " + "fallback is exercised in every compatible cell", +) def test_get_version_passthrough_when_installed(monkeypatch): """On a normal install the real version is returned unchanged (no fallback).""" mod = importlib.import_module("weaviate.proto.v1") From 0f2f6da1e5e1788809d7871e427451d3127440ce Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Tue, 9 Jun 2026 22:24:34 +0300 Subject: [PATCH 05/27] fix(grpc-web): guard grpc-web mode (shim+async required), tighten timeout/fallback Addresses the second Copilot review on #2056: - base.py: grpc_path_prefix now fails fast in _grpc_channel when used on a sync client or when the weaviate-python-grpc-web shim is not active, instead of silently building a native grpcio channel that ignores the prefix. - helpers.py: connect_to_custom (sync) rejects a non-empty grpc_path_prefix and points to use_async_with_custom; the docstring is clarified that grpc-web is async-only. - _channel.py: _encode_timeout rounds the grpc-timeout up (math.ceil) so we never advertise a shorter deadline than requested. - proto/v1/__init__.py: the grpcio metadata fallback is restricted to Emscripten, so a broken/partial grpcio install on a normal platform surfaces as PackageNotFoundError instead of being masked by a fallback stub version. Tests added/updated (sync/no-shim rejection, off-emscripten raise, grpc-timeout round-up). End-to-end against a vanguard transcoder on a shared host:port still passes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/weaviate_grpc_web/_channel.py | 7 ++- packages/grpc-web/tests/test_transport.py | 9 ++++ proto_test/test_proto.py | 20 +++++++++ test/test_connection_params.py | 45 +++++++++++++++++++ weaviate/connect/base.py | 21 +++++++-- weaviate/connect/helpers.py | 16 ++++--- weaviate/proto/v1/__init__.py | 11 +++-- 7 files changed, 115 insertions(+), 14 deletions(-) diff --git a/packages/grpc-web/src/weaviate_grpc_web/_channel.py b/packages/grpc-web/src/weaviate_grpc_web/_channel.py index 853e4fc0b..cb16f7bc2 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_channel.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_channel.py @@ -14,6 +14,7 @@ import asyncio import base64 +import math import urllib.parse from typing import Any, Callable, Dict, Optional @@ -37,10 +38,12 @@ def get_sender() -> Sender: def _encode_timeout(seconds: float) -> str: """Encode a timeout as a grpc-timeout header value (````).""" - millis = max(1, int(seconds * 1000)) + # Round up so we never advertise a shorter deadline than requested (which would risk + # premature server-side cancellation); grpc-timeout takes a positive integer + unit. + millis = max(1, math.ceil(seconds * 1000)) if millis < 100_000_000: return f"{millis}m" - return f"{max(1, int(seconds))}S" + return f"{max(1, math.ceil(seconds))}S" def _fold_metadata(headers: Dict[str, str], metadata: Any) -> None: diff --git a/packages/grpc-web/tests/test_transport.py b/packages/grpc-web/tests/test_transport.py index 753f227de..f931fa8e1 100644 --- a/packages/grpc-web/tests/test_transport.py +++ b/packages/grpc-web/tests/test_transport.py @@ -180,6 +180,15 @@ def test_malformed_grpc_status_maps_to_internal(): assert excinfo.value.code() is StatusCode.INTERNAL +def test_grpc_timeout_header_rounds_up(): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + # 123.4ms must round UP to 124ms (never advertise a shorter deadline than requested). + asyncio.run(mc(b"q", timeout=0.1234)) + assert sender.calls[0][1]["grpc-timeout"] == "124m" + + def test_close_is_awaitable_noop(): channel = _channel(FakeSender()) assert asyncio.run(channel.close()) is None diff --git a/proto_test/test_proto.py b/proto_test/test_proto.py index 7b089d875..54dd89a15 100644 --- a/proto_test/test_proto.py +++ b/proto_test/test_proto.py @@ -60,12 +60,32 @@ def raises(pkg: str) -> str: raise PackageNotFoundError(pkg) monkeypatch.setattr(mod, "metadata_version", raises) + monkeypatch.setattr("sys.platform", "emscripten") assert str(mod.get_version("grpcio")) == "1.72.1" with pytest.raises(PackageNotFoundError): mod.get_version("protobuf") +@pytest.mark.skipif( + _INCOMPATIBLE_GRPC_PB, + reason="weaviate.proto.v1 cannot be imported with an incompatible grpcio/protobuf " + "pair (CI version-gate matrix); the gate is covered by test_proto_import and the " + "fallback is exercised in every compatible cell", +) +def test_grpcio_missing_metadata_raises_off_emscripten(monkeypatch): + """Off Emscripten, missing grpcio metadata surfaces instead of being masked.""" + mod = importlib.import_module("weaviate.proto.v1") + + def raises(pkg: str) -> str: + raise PackageNotFoundError(pkg) + + monkeypatch.setattr(mod, "metadata_version", raises) + monkeypatch.setattr("sys.platform", "linux") + with pytest.raises(PackageNotFoundError): + mod.get_version("grpcio") + + @pytest.mark.skipif( _INCOMPATIBLE_GRPC_PB, reason="weaviate.proto.v1 cannot be imported with an incompatible grpcio/protobuf " diff --git a/test/test_connection_params.py b/test/test_connection_params.py index 69b07f5a4..041079cfc 100644 --- a/test/test_connection_params.py +++ b/test/test_connection_params.py @@ -3,6 +3,7 @@ import weaviate.connect.base as base_mod from weaviate.connect.base import ConnectionParams +from weaviate.exceptions import WeaviateInvalidInputError def test_same_host_port_raises_without_prefix() -> None: @@ -88,6 +89,8 @@ def fake_insecure_channel(target, options=None, **kwargs): return "CHANNEL" monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) + # grpc-web mode requires the shim to be active; simulate it being installed. + monkeypatch.setattr(base_mod.grpc, "__weaviate_grpc_web_shim__", True, raising=False) params = ConnectionParams.from_params( http_host="localhost", @@ -105,6 +108,48 @@ def fake_insecure_channel(target, options=None, **kwargs): assert ("grpc-web.path_prefix", "/grpc-web") in captured["options"] +def _grpc_web_params() -> ConnectionParams: + return ConnectionParams.from_params( + http_host="localhost", + http_port=8090, + http_secure=False, + grpc_host="localhost", + grpc_port=8090, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + + +def test_grpc_channel_rejects_prefix_without_shim(monkeypatch) -> None: + # No grpc-web shim active -> must fail fast instead of silently building a native + # grpcio channel that ignores the prefix. + monkeypatch.delattr(base_mod.grpc, "__weaviate_grpc_web_shim__", raising=False) + with pytest.raises(WeaviateInvalidInputError, match="weaviate-python-grpc-web"): + _grpc_web_params()._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) + + +def test_grpc_channel_rejects_prefix_for_sync_client() -> None: + # grpc-web is async-only; a sync channel with a prefix must be rejected. + with pytest.raises(WeaviateInvalidInputError, match="async"): + _grpc_web_params()._grpc_channel(proxies={}, grpc_msg_size=None, is_async=False) + + +def test_connect_to_custom_rejects_grpc_web_prefix() -> None: + # The synchronous helper must reject grpc-web up front (before connecting). + import weaviate + + with pytest.raises(WeaviateInvalidInputError, match="async-only"): + weaviate.connect_to_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=8080, + grpc_secure=False, + grpc_path_prefix="/grpc-web", + ) + + def test_grpc_channel_omits_option_without_prefix(monkeypatch) -> None: captured: dict = {} diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index df2c79a28..eaf72f73e 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -9,6 +9,7 @@ from pydantic import BaseModel, field_validator, model_validator from weaviate.config import GrpcConfig, Proxies +from weaviate.exceptions import WeaviateInvalidInputError from weaviate.types import NUMBER from weaviate.util import is_weaviate_domain @@ -160,10 +161,24 @@ def _grpc_channel( if grpc_config is not None and grpc_config.channel_options is not None: options.extend(grpc_config.channel_options) - # In grpc-web mode, forward the base-path prefix to the transport via channel - # options (consumed by the weaviate-python-grpc-web shim). Not added for native - # gRPC, so the native channel options stay byte-for-byte unchanged. + # grpc-web mode (prefix set): only valid for an async client, and only when the + # weaviate-python-grpc-web shim has replaced the grpc module (it consumes the + # grpc-web.path_prefix option). Fail fast otherwise — a native grpcio channel + # would silently ignore the option and route over native gRPC, a confusing + # misconfiguration. Nothing is added for native gRPC, so its channel options stay + # byte-for-byte unchanged. if (prefix := self._grpc_web_path_prefix) != "": + if not is_async: + raise WeaviateInvalidInputError( + "grpc_path_prefix (grpc-web) is only supported for async clients; " + "use use_async_with_custom(...) / WeaviateAsyncClient" + ) + if not getattr(grpc, "__weaviate_grpc_web_shim__", False): + raise WeaviateInvalidInputError( + "grpc_path_prefix enables grpc-web, which requires the " + "'weaviate-python-grpc-web' package (it installs a grpc shim before " + "'import weaviate'); it is not active in this environment" + ) options.append(("grpc-web.path_prefix", prefix)) if is_async: diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index c9aed194b..bb9e3d771 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -17,6 +17,7 @@ from weaviate.config import AdditionalConfig from weaviate.connect.base import ConnectionParams, ProtocolParams from weaviate.embedded import WEAVIATE_VERSION, EmbeddedOptions +from weaviate.exceptions import WeaviateInvalidInputError from weaviate.util import docstring_deprecated from weaviate.validator import _validate_input, _ValidateArgument from weaviate.warnings import _Warnings @@ -313,11 +314,11 @@ def connect_to_custom( a bearer token, in which case use `weaviate.classes.init.Auth.bearer_token()`, a client secret, in which case use `weaviate.classes.init.Auth.client_credentials()` or a username and password, in which case use `weaviate.classes.init.Auth.client_password()`. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. - grpc_path_prefix: Optional base-path prefix for a grpc-web endpoint served on the - same host:port as REST (e.g. "/grpc-web"). When set, gRPC requests are sent - over grpc-web to ``://:/...`` and sharing - the REST host:port is allowed. Requires the ``weaviate-python-grpc-web`` - package. Defaults to None (native gRPC). + grpc_path_prefix: grpc-web base-path prefix. grpc-web is async-only, so it is NOT + supported by the synchronous ``connect_to_custom`` — passing a non-empty value + raises ``WeaviateInvalidInputError``. Use + ``use_async_with_custom(..., grpc_path_prefix=...)`` instead. Defaults to None + (native gRPC). Returns: The client connected to the instance with the required parameters set appropriately. @@ -350,6 +351,11 @@ def connect_to_custom( True >>> # The connection is automatically closed when the context is exited. """ + if grpc_path_prefix: + raise WeaviateInvalidInputError( + "grpc_path_prefix enables grpc-web, which is async-only; use " + "use_async_with_custom(...) instead of connect_to_custom(...)" + ) return __connect( WeaviateClient( ConnectionParams.from_params( diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index a3821fed5..f20e52e05 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -1,3 +1,4 @@ +import sys import warnings @@ -19,16 +20,18 @@ # This happens under Pyodide/Emscripten, where grpcio has no wheel and is excluded # via the `sys_platform != "emscripten"` marker in setup.cfg; the grpc module itself # is provided there by a pure-Python shim (see the weaviate-python-grpc-web package). -# On every normal install grpcio's metadata is present and the real version is used, -# so this branch is not taken. Restricted to grpcio so that a genuinely missing -# protobuf (which is required and pure-Python under Pyodide) is never masked. +# On every normal install grpcio's metadata is present and the real version is used, so +# this branch is not taken. Restricted to grpcio AND to Emscripten, so that a broken or +# partial grpcio install on a normal platform (metadata missing) still surfaces as +# PackageNotFoundError instead of silently selecting a fallback stub, and so a genuinely +# missing protobuf (required and pure-Python under Pyodide) is never masked. _GRPCIO_FALLBACK_VERSION = "1.72.1" def get_version(pkg: str) -> version.Version: try: return version.parse(metadata_version(pkg)) except PackageNotFoundError: - if pkg == "grpcio": + if pkg == "grpcio" and sys.platform == "emscripten": return version.parse(_GRPCIO_FALLBACK_VERSION) raise From 3669363f74c1b3f039d1f54923f6330931f6fd31 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:05:11 +0300 Subject: [PATCH 06/27] feat(grpc-web): route httpx REST calls over fetch under Emscripten First live Pyodide run showed the gRPC path working while every REST call (is_ready, schema ops, batch references) failed: httpx/httpcore open raw sockets, which do not exist under WASM, surfacing only as "Connection to Weaviate failed. Details: " with empty details. Adds _httpx_fetch.py: patches httpx.AsyncHTTPTransport.handle_async_request to send requests via pyodide.http.pyfetch (buffered responses, fetch-managed headers stripped, best-effort AbortSignal timeout). Installed by the package bootstrap under Emscripten only, with install_fetch_transport(force=True) for CPython testing. Verified live from Pyodide-in-Node against a WCD dev cluster behind a grpc-web transcoder: is_ready, collection create/delete, insert_many with server-side vectorization, aggregate, and near_text all pass. Co-Authored-By: Claude Fable 5 --- .../src/weaviate_grpc_web/__init__.py | 11 +++ .../src/weaviate_grpc_web/_httpx_fetch.py | 95 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py diff --git a/packages/grpc-web/src/weaviate_grpc_web/__init__.py b/packages/grpc-web/src/weaviate_grpc_web/__init__.py index 79a6cb1d8..542d075bd 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/__init__.py +++ b/packages/grpc-web/src/weaviate_grpc_web/__init__.py @@ -26,6 +26,8 @@ __all__ = [ "install", "is_installed", + "install_fetch_transport", + "is_fetch_transport_installed", "set_sender", "make_httpx_sender", "GrpcWebChannel", @@ -40,6 +42,11 @@ def _bootstrap() -> None: # effect. ``setdefault`` lets a user override it explicitly. os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") install() + # The REST path needs fetch too: httpx/httpcore open raw sockets, which do + # not exist under WASM. Imported lazily so CPython imports stay light. + from ._httpx_fetch import install_fetch_transport + + install_fetch_transport() _bootstrap() @@ -48,4 +55,8 @@ def _bootstrap() -> None: # ``._shim`` (not via ``sys.modules['grpc']``), so importing them is safe regardless of # whether the shim was installed. from ._channel import GrpcWebChannel, set_sender # noqa: E402 +from ._httpx_fetch import ( # noqa: E402 + install_fetch_transport, + is_fetch_transport_installed, +) from ._sender import make_httpx_sender # noqa: E402 diff --git a/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py b/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py new file mode 100644 index 000000000..8a84e5d11 --- /dev/null +++ b/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py @@ -0,0 +1,95 @@ +"""fetch-based httpx transport for Pyodide/Emscripten. + +The base client's REST path uses ``httpx.AsyncClient`` with explicit +``httpx.AsyncHTTPTransport`` mounts (``weaviate/connect/v4.py``). httpcore opens raw +sockets, which do not exist under WASM, so without this module every REST call +(``is_ready``, collection config, batch references, …) fails with an empty connection +error even though the grpc-web data path works. + +Installing reroutes ``AsyncHTTPTransport.handle_async_request`` through the browser's +``fetch`` via ``pyodide.http.pyfetch`` — the same install-globally-under-Emscripten +philosophy as the grpc shim in ``_shim.py``. Responses are fully buffered, which matches +how the base client consumes them (JSON bodies, no streaming). +""" + +import sys +from typing import Dict + +import httpx + +_installed = False + +# Hop-by-hop / connection-managed headers that the browser's fetch controls itself. +# Browsers silently drop forbidden headers, but Node's undici (used by the CPython/Node +# test path) rejects some of them outright, so strip them before handing off. +_FETCH_MANAGED_HEADERS = { + "host", + "connection", + "accept-encoding", + "content-length", + "transfer-encoding", +} + + +async def _read_request_body(request: httpx.Request) -> bytes: + try: + return request.content + except httpx.RequestNotRead: + return await request.aread() + + +async def _fetch_handle_async_request( + self: httpx.AsyncHTTPTransport, request: httpx.Request +) -> httpx.Response: + from pyodide.http import pyfetch # type: ignore[import-not-found] + + headers: Dict[str, str] = { + k: v for k, v in request.headers.items() if k.lower() not in _FETCH_MANAGED_HEADERS + } + kwargs: Dict[str, object] = {} + body = await _read_request_body(request) + if body: + # fetch rejects GET/HEAD requests that carry a body + kwargs["body"] = body + + timeouts = request.extensions.get("timeout") or {} + timeout = timeouts.get("read") or timeouts.get("connect") or timeouts.get("pool") + if timeout: + try: + from js import AbortSignal # type: ignore[import-not-found] + + kwargs["signal"] = AbortSignal.timeout(int(timeout * 1000)) + except Exception: # pragma: no cover - AbortSignal.timeout availability varies + pass + + response = await pyfetch(str(request.url), method=request.method, headers=headers, **kwargs) + data = await response.bytes() + try: + resp_headers = dict(response.headers) + except Exception: # pragma: no cover - header shape varies across Pyodide versions + resp_headers = {} + return httpx.Response( + status_code=int(response.status), + headers=resp_headers, + content=data, + request=request, + ) + + +def install_fetch_transport(force: bool = False) -> None: + """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. + + Installs only under Emscripten unless ``force=True`` (CPython testing, where a + ``pyodide`` stub must be importable). Idempotent. + """ + global _installed + if _installed: + return + if not force and sys.platform != "emscripten": + return + httpx.AsyncHTTPTransport.handle_async_request = _fetch_handle_async_request # type: ignore[method-assign] + _installed = True + + +def is_fetch_transport_installed() -> bool: + return _installed From b8ac294dd1df6ed885904b51458bd19a1581bd24 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:18:38 +0300 Subject: [PATCH 07/27] fix: make the async client safe under WASM/Pyodide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Threads, subprocesses, and blocking sleeps do not exist under Emscripten, and transport errors often stringify to '' — audited every async runtime path and fixed the failures: - OIDC: refresh tokens with an asyncio task instead of the TokenRefresh daemon thread + event-loop sidecar thread (both crash connect() under WASM with "can't start new thread" after the token fetch already succeeded). close() cancels the task; a failed or retried connect() cancels the previous refresher instead of orphaning it against the IdP. The sync client keeps the thread-based path unchanged. - batch: batch.stream() fails fast with a typed error over grpc-web (bidi streaming impossible) instead of silently dropping objects; flush() raises the background failure instead of spinning forever; _wait() preserves partial results before raising. - embedded: raise an explicit "not supported under WebAssembly/Pyodide" error instead of the false "processes are already listening" produced by Emscripten's lazy socket emulation. - error surfacing: include the exception type where str(e) can be empty (the live "Connection to Weaviate failed. Details: " bug); stop rewriting unrelated RuntimeErrors as "client is closed"; chain OIDC discovery errors; log is_ready/is_live failures via logger instead of print(); tolerate OSError in the best-effort PyPI version check (CSP blocks pypi.org in browsers and used to fail connect()). - wait_for_weaviate (async): asyncio.sleep instead of time.sleep, which stalls the event loop. - dedupe the grpc-web shim marker sniff into _grpc_web_shim_active(). Co-Authored-By: Claude Fable 5 --- mock_tests/test_auth.py | 85 +++++++++ test/test_batch_async.py | 76 ++++++++ test/test_wasm_compat.py | 73 +++++++ weaviate/client_executor.py | 7 +- weaviate/collections/batch/async_.py | 22 ++- weaviate/connect/base.py | 13 +- weaviate/connect/v4.py | 273 ++++++++++++++++++--------- weaviate/embedded.py | 9 + 8 files changed, 464 insertions(+), 94 deletions(-) create mode 100644 test/test_batch_async.py create mode 100644 test/test_wasm_compat.py diff --git a/mock_tests/test_auth.py b/mock_tests/test_auth.py index 192f0eb6d..fc07c904e 100644 --- a/mock_tests/test_auth.py +++ b/mock_tests/test_auth.py @@ -84,6 +84,44 @@ def test_client_credentials(weaviate_auth_mock: HTTPServer, start_grpc_server: g weaviate_auth_mock.check_assertions() +@pytest.mark.asyncio +async def test_client_credentials_refresh_async( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server +) -> None: + """Test the refresh_session branch of the async token refresher. + + Client-credentials tokens carry no refresh token, so the refresher must get a whole + new token from the saved credentials. + """ + token_requests = 0 + + def handler(request: Request) -> Response: + nonlocal token_requests + token_requests += 1 + return Response( + json.dumps({"access_token": ACCESS_TOKEN, "expires_in": 1}), + content_type="application/json", + ) + + weaviate_auth_mock.expect_request("/auth").respond_with_handler(handler) + weaviate_auth_mock.expect_request( + "/v1/schema", headers={"Authorization": "Bearer " + ACCESS_TOKEN} + ).respond_with_json({"classes": []}) + + async with weaviate.use_async_with_local( + host=MOCK_IP, + port=MOCK_PORT, + grpc_port=MOCK_PORT_GRPC, + auth_credentials=weaviate.auth.AuthClientCredentials( + client_secret=CLIENT_SECRET, scope=SCOPE + ), + ) as client: + await client.collections.list_all() + first = token_requests + await asyncio.sleep(3) # refresh interval is max(expires_in - 30, 1) -> 1s + assert token_requests > first # a fresh token was fetched with the credentials + + @pytest.mark.parametrize("header_name", ["Authorization", "authorization"]) def test_auth_header_priority( recwarn, weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server, header_name: str @@ -183,6 +221,53 @@ async def test_refresh_async( weaviate_auth_mock.check_assertions() +@pytest.mark.asyncio +async def test_async_auth_starts_no_threads( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server +) -> None: + """The async client must refresh tokens with an asyncio task, not threads. + + Under WASM/Pyodide threads cannot start at all, so the TokenRefresh daemon thread + and the event-loop sidecar thread would make every async OIDC flow crash connect(). + """ + import threading + + weaviate_auth_mock.expect_request( + "/v1/schema", headers={"Authorization": "Bearer " + ACCESS_TOKEN} + ).respond_with_json({"classes": []}) + weaviate_auth_mock.expect_request("/auth").respond_with_json( + { + "access_token": ACCESS_TOKEN, + "expires_in": 500, + "refresh_token": REFRESH_TOKEN, + } + ) + + # compare thread OBJECTS, not names: earlier sync tests leave stale TokenRefresh + # daemon threads alive, which would mask a regression in a name-set comparison + threads_before = set(threading.enumerate()) + tasks_before = asyncio.all_tasks() + async with weaviate.use_async_with_local( + host=MOCK_IP, + port=MOCK_PORT, + grpc_port=MOCK_PORT_GRPC, + auth_credentials=weaviate.auth.AuthBearerToken( + ACCESS_TOKEN, refresh_token=REFRESH_TOKEN, expires_in=500 + ), + ) as client: + await client.collections.list_all() + new_thread_names = {t.name for t in set(threading.enumerate()) - threads_before} + assert "TokenRefresh" not in new_thread_names + assert "eventLoop" not in new_thread_names + refresh_tasks = [ + t for t in asyncio.all_tasks() - tasks_before if "token_refresh" in repr(t.get_coro()) + ] + assert len(refresh_tasks) == 1 # the refresher runs as an asyncio task instead + # ... and close() must cancel it, not leak it (one wait for the cancellation to land) + await asyncio.wait(refresh_tasks, timeout=1) + assert refresh_tasks[0].done() + + def test_refresh_of_refresh(weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server) -> None: """Test that refresh tokens are used to get a new refresh token token.""" weaviate_auth_mock.expect_request( diff --git a/test/test_batch_async.py b/test/test_batch_async.py new file mode 100644 index 000000000..4c0d59726 --- /dev/null +++ b/test/test_batch_async.py @@ -0,0 +1,76 @@ +"""Unit tests for the async batch-stream failure handling. + +These pin three behaviors added for WASM/background-failure robustness without needing +a cluster: the grpc-web fail-fast in _start, flush() raising instead of spinning +forever, and _wait() preserving partial results while still raising. +""" + +import asyncio + +import grpc +import pytest + +from weaviate.collections.batch.async_ import _BatchBaseAsync +from weaviate.collections.batch.base import _BatchDataWrapper +from weaviate.exceptions import WeaviateBatchStreamError + + +def _bare_batch(**mangled) -> _BatchBaseAsync: + batch = object.__new__(_BatchBaseAsync) + for name, value in mangled.items(): + setattr(batch, f"_BatchBaseAsync__{name}", value) + return batch + + +def test_start_fails_fast_when_grpc_web_shim_active(monkeypatch) -> None: + # over grpc-web the BatchStream RPC would die inside the background tasks (silent + # drop / endless flush); _start must raise before any task is created + monkeypatch.setattr(grpc, "__weaviate_grpc_web_shim__", True, raising=False) + batch = _bare_batch() # the guard runs before any attribute access + with pytest.raises(WeaviateBatchStreamError, match="insert_many"): + asyncio.run(batch._start()) + + +def test_flush_raises_background_exception_instead_of_hanging() -> None: + # with dead background tasks nothing drains the queues; flush used to spin on + # asyncio.sleep(0.01) forever + batch = _bare_batch( + bg_exception=RuntimeError("boom"), + batch_objects=[object()], + batch_references=[], + ) + + async def flush_with_deadline() -> None: + await asyncio.wait_for(batch.flush(), timeout=2) + + with pytest.raises(RuntimeError, match="boom"): + asyncio.run(flush_with_deadline()) + + +def test_wait_copies_partial_results_before_raising() -> None: + # a user catching the background failure must still see what failed + class FakeBgTasks: + async def gather(self, timeout=None) -> None: + return None + + class FakeTimeouts: + insert = 1 + + class FakeConnection: + timeout_config = FakeTimeouts() + + partial = _BatchDataWrapper() + partial.failed_objects = ["sentinel-failure"] # type: ignore[list-item] + backup = _BatchDataWrapper() + + batch = _bare_batch( + bg_exception=RuntimeError("boom"), + bg_tasks=FakeBgTasks(), + connection=FakeConnection(), + results_for_wrapper=partial, + results_for_wrapper_backup=backup, + ) + + with pytest.raises(RuntimeError, match="boom"): + asyncio.run(batch._wait()) + assert backup.failed_objects == ["sentinel-failure"] diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py new file mode 100644 index 000000000..dc3355bc9 --- /dev/null +++ b/test/test_wasm_compat.py @@ -0,0 +1,73 @@ +"""Unit tests for WASM/Pyodide-compatibility behavior that runs on CPython too. + +Under Emscripten there are no subprocesses and no threads, and transport errors often +stringify to '' — these tests pin the guards and error-surfacing added for that +environment without needing a browser. +""" + +import sys + +import pytest +from httpx import ConnectError, ReadTimeout + +from weaviate.connect.v4 import _ConnectionBase, _exc_detail +from weaviate.embedded import _EmbeddedBase +from weaviate.exceptions import ( + WeaviateClosedClientError, + WeaviateConnectionError, + WeaviateStartUpError, + WeaviateTimeoutError, +) + + +def test_embedded_raises_explicit_error_under_emscripten(monkeypatch) -> None: + # without the guard, the Emscripten socket emulation makes the port probe + # "succeed" and embedded misreports that Weaviate is already listening + monkeypatch.setattr(sys, "platform", "emscripten") + with pytest.raises(WeaviateStartUpError, match="WebAssembly/Pyodide"): + _EmbeddedBase.check_supported_platform() + + +def test_embedded_platform_check_passes_on_supported_platforms() -> None: + assert sys.platform != "emscripten" + _EmbeddedBase.check_supported_platform() # must not raise on this dev platform + + +def _handle_exceptions(e: Exception, error_msg: str = "") -> None: + conn = object.__new__(_ConnectionBase) + # keep the bare instance's __del__ quiet (it checks these for unclosed connections) + conn._client = None + conn._grpc_channel = None + getattr(conn, "_ConnectionBase__handle_exceptions")(e, error_msg) # noqa: B009 + + +def test_httpx_closed_client_runtime_error_maps_to_closed_client() -> None: + # the exact message httpx raises for a closed AsyncClient/Client + with pytest.raises(WeaviateClosedClientError): + _handle_exceptions(RuntimeError("Cannot send a request, as the client has been closed.")) + + +def test_unrelated_runtime_error_is_not_rewritten_as_closed_client() -> None: + # Emscripten's canonical thread failure must propagate as-is, not as a misleading + # 'client is closed - run client.connect()' + with pytest.raises(RuntimeError, match="can't start new thread"): + _handle_exceptions(RuntimeError("can't start new thread")) + + +def test_connect_error_message_includes_exception_type() -> None: + # str(httpx.ConnectError('')) == '' — the type name must still surface + with pytest.raises(WeaviateConnectionError) as excinfo: + _handle_exceptions(ConnectError("")) + assert "ConnectError" in str(excinfo.value) + + +def test_read_timeout_message_includes_context_and_detail() -> None: + with pytest.raises(WeaviateTimeoutError) as excinfo: + _handle_exceptions(ReadTimeout(""), error_msg="Meta endpoint") + assert "Meta endpoint" in str(excinfo.value) + assert "ReadTimeout" in str(excinfo.value) + + +def test_exc_detail_formats_empty_and_nonempty_strs() -> None: + assert _exc_detail(ValueError("boom")) == "ValueError: boom" + assert _exc_detail(ConnectError("")) == "ConnectError('')" diff --git a/weaviate/client_executor.py b/weaviate/client_executor.py index 3125fd9cd..217f0d7a5 100644 --- a/weaviate/client_executor.py +++ b/weaviate/client_executor.py @@ -16,6 +16,7 @@ from weaviate.collections.classes.internal import _GQLEntryReturnType, _RawGQLReturn from weaviate.integrations import _Integrations +from weaviate.logger import logger from .auth import AuthCredentials from .config import AdditionalConfig @@ -164,7 +165,7 @@ def resp(_: None) -> bool: return True def exc(e: Exception) -> bool: - print(e) + logger.warning(f"gRPC health check failed: {e!r}") return False return executor.execute( @@ -196,7 +197,7 @@ async def await_grpc_result() -> bool: return grpc_result def exc(e: Exception) -> bool: - print(e) + logger.warning(f"is_live check failed: {e!r}") return False return cast( @@ -214,7 +215,7 @@ def resp(res: Response) -> bool: return res.status_code == 200 def exc(e: Exception) -> bool: - print(e) + logger.warning(f"is_ready check failed: {e!r}") return False return executor.execute( diff --git a/weaviate/collections/batch/async_.py b/weaviate/collections/batch/async_.py index 8b997586c..fe2dff546 100644 --- a/weaviate/collections/batch/async_.py +++ b/weaviate/collections/batch/async_.py @@ -37,6 +37,7 @@ ReferenceToMulti, ) from weaviate.collections.classes.types import WeaviateProperties +from weaviate.connect.base import _grpc_web_shim_active from weaviate.connect.executor import aresult from weaviate.connect.v4 import ConnectionAsync from weaviate.exceptions import ( @@ -133,6 +134,15 @@ def __all_tasks_alive(self) -> bool: return self.__bg_tasks is not None and self.__bg_tasks.all_alive() async def _start(self): + if _grpc_web_shim_active(): + # fail fast and loud: over grpc-web the BatchStream RPC raises inside the + # background tasks, where it would otherwise surface as a silent drop or a + # never-ending flush() + raise WeaviateBatchStreamError( + "batch.stream() requires bidirectional gRPC streaming, which is not " + "possible over grpc-web/fetch (WebAssembly/Pyodide). Use " + "collection.data.insert_many() instead." + ) self.__number_of_nodes = await self.__cluster.get_number_of_nodes() async def loop_wrapper() -> None: @@ -192,7 +202,8 @@ async def _wait(self) -> None: "Background batch tasks did not terminate after forced shutdown." ) from e - # copy the results to the public results + # copy the results to the public results — even on failure, so the user can + # still inspect batch.results / batch.failed_objects after catching self.__results_for_wrapper_backup.results = self.__results_for_wrapper.results self.__results_for_wrapper_backup.failed_objects = self.__results_for_wrapper.failed_objects self.__results_for_wrapper_backup.failed_references = ( @@ -202,6 +213,11 @@ async def _wait(self) -> None: self.__results_for_wrapper.imported_shards ) + if self.__bg_exception is not None: + # surface background-task failures instead of returning partial results + # as if the batch had succeeded + raise self.__bg_exception + async def _shutdown(self) -> None: self.__is_stopped.set() @@ -531,6 +547,10 @@ async def flush(self) -> None: """Flush the batch queue and wait for all requests to be finished.""" # bg thread is sending objs+refs automatically, so simply wait for everything to be done while len(self.__batch_objects) > 0 or len(self.__batch_references) > 0: + if self.__bg_exception is not None: + # the background tasks died; nothing will drain the queues, so waiting + # any longer would hang forever + raise self.__bg_exception await asyncio.sleep(0.01) async def _add_object( diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index eaf72f73e..f64c026c5 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -21,6 +21,17 @@ MAX_GRPC_MESSAGE_LENGTH = 104858000 # 10mb, needs to be synchronized with GRPC server +def _grpc_web_shim_active() -> bool: + """Whether the 'weaviate-python-grpc-web' shim has replaced the grpc module. + + The shim (used under WASM/Pyodide, where there is no grpcio wheel) routes unary RPCs + over grpc-web/fetch and cannot do bidirectional streaming. The marker attribute is + the documented contract between the two packages — keep all sniffs going through + this helper. + """ + return getattr(grpc, "__weaviate_grpc_web_shim__", False) is True + + class ProtocolParams(BaseModel): host: str port: int @@ -173,7 +184,7 @@ def _grpc_channel( "grpc_path_prefix (grpc-web) is only supported for async clients; " "use use_async_with_custom(...) / WeaviateAsyncClient" ) - if not getattr(grpc, "__weaviate_grpc_web_shim__", False): + if not _grpc_web_shim_active(): raise WeaviateInvalidInputError( "grpc_path_prefix enables grpc-web, which requires the " "'weaviate-python-grpc-web' package (it installs a grpc shim before " diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index 56ece8ca2..f674ff74a 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import time from copy import copy from dataclasses import dataclass, field @@ -109,6 +110,16 @@ PERMISSION_DENIED = "PERMISSION_DENIED" +def _exc_detail(e: BaseException) -> str: + """Format an exception for user-facing messages. + + ``str()`` of transport errors is frequently empty (e.g. ``httpx.ConnectError``), + which produces messages like 'Details: ' with nothing after them — always include + the exception type. + """ + return f"{type(e).__name__}: {e}" if str(e) else repr(e) + + @dataclass class _ExpectedStatusCodes: ok_in: Union[List[int], int] @@ -153,6 +164,8 @@ def __init__( self._connected = False self._skip_init_checks = skip_init_checks self._grpc_config = grpc_config + self._shutdown_background_event: Optional[Event] = None + self.__token_refresh_task: Optional["asyncio.Task[None]"] = None client_type = "sync" if isinstance(self, ConnectionSync) else "async" embedded_suffix = "-embedded" if self.embedded_db is not None else "" @@ -408,8 +421,8 @@ async def get_oidc() -> None: response = await client.get(oidc_url, timeout=self.timeout_config.init) except Exception as e: raise WeaviateConnectionError( - f"Error: {e}. \nIs Weaviate running and reachable at {self.url}?" - ) + f"Error: {_exc_detail(e)}. \nIs Weaviate running and reachable at {self.url}?" + ) from e res = self.__process_oidc_response(response, auth_client_secret, oidc_url, colour) if isinstance(res, Awaitable): return await res @@ -423,8 +436,8 @@ async def get_oidc() -> None: response = client.get(oidc_url, timeout=self.timeout_config.init) except Exception as e: raise WeaviateConnectionError( - f"Error: {e}. \nIs Weaviate running and reachable at {self.url}?" - ) + f"Error: {_exc_detail(e)}. \nIs Weaviate running and reachable at {self.url}?" + ) from e res = self.__process_oidc_response(response, auth_client_secret, oidc_url, colour) assert not isinstance(res, Awaitable) return res @@ -530,18 +543,37 @@ def _create_background_token_refresh(self, _auth: Optional[_Auth] = None) -> Non if "refresh_token" not in self._client.token and _auth is None: return - # make an event loop sidecar thread for running async token refreshing - event_loop = ( - _EventLoopSingleton.get_instance() - if isinstance(self._client, AsyncOAuth2Client) - else None - ) + # a previous connect() may have left a refresher behind (e.g. a retry after a + # partially failed connect); stop it before replacing the shutdown event, or it + # would keep refreshing concurrently forever + self._cancel_background_token_refresh() expires_in: int = self._client.token.get( "expires_in", 60 ) # use 1minute as token lifetime if not supplied self._shutdown_background_event = Event() + if isinstance(self._client, AsyncOAuth2Client): + try: + loop: Optional[asyncio.AbstractEventLoop] = asyncio.get_running_loop() + except RuntimeError: + loop = None + if loop is not None: + # The async colour refreshes on its own already-running loop: threads + # cannot start under WASM/Pyodide and are unnecessary here anyway. + self.__token_refresh_task = loop.create_task( + self.__periodic_token_refresh_async(expires_in, _auth) + ) + return + + # sync colour (or async without a running loop): refresh on a daemon thread, + # with an event loop sidecar thread for running async token refreshing + event_loop = ( + _EventLoopSingleton.get_instance() + if isinstance(self._client, AsyncOAuth2Client) + else None + ) + def refresh_token() -> None: if isinstance(self._client, AsyncOAuth2Client): assert event_loop is not None @@ -603,6 +635,49 @@ def periodic_refresh_token(refresh_time: int, _auth: Optional[_Auth]) -> None: ) demon.start() + def _cancel_background_token_refresh(self) -> None: + """Stop the token refresher, whichever form it took. + + Sets the shutdown event (observed by the sync colour's daemon thread at its next + wake-up) and cancels the async colour's refresh task immediately. + """ + if self._shutdown_background_event is not None: + self._shutdown_background_event.set() + if self.__token_refresh_task is not None: + self.__token_refresh_task.cancel() + self.__token_refresh_task = None + + async def __periodic_token_refresh_async( + self, refresh_time: int, _auth: Optional[_Auth] + ) -> None: + """Thread-free equivalent of ``periodic_refresh_token`` for the async colour. + + Cancelled by ``close('async')``. + """ + while ( + self._shutdown_background_event is not None + and not self._shutdown_background_event.is_set() + ): + # use refresh token when available + await asyncio.sleep(max(refresh_time, 1)) + try: + client = self._client + if not isinstance(client, AsyncOAuth2Client): + continue + if "refresh_token" in client.token: + client.token = await client.refresh_token(url=client.metadata["token_endpoint"]) + else: + # client credentials usually does not contain a refresh token => get a + # new token using the saved credentials + assert _auth is not None + new_session = await _Auth.aresult(_auth.get_auth_session()) + client.token = await new_session.fetch_token() + refresh_time = client.token.get("expires_in", 60) - 30 + except HTTPError as exc: + # retry again after one second, might be an unstable connection + refresh_time = 1 + _Warnings.token_refresh_failed(exc) + def __get_latest_headers(self) -> Dict[str, str]: if "authorization" in self._headers: return self._headers @@ -648,14 +723,23 @@ def __get_timeout( ) def __handle_exceptions(self, e: Exception, error_msg: str) -> None: - if isinstance(e, RuntimeError): + # httpx raises a bare RuntimeError('Cannot send a request, as the client has been + # closed.'); match its message so unrelated RuntimeErrors (e.g. Emscripten's + # "can't start new thread") are not rewritten into a misleading 'client is + # closed' error. + if isinstance(e, RuntimeError) and "client has been closed" in str(e): raise WeaviateClosedClientError() from e if isinstance(e, ConnectError): - raise WeaviateConnectionError(error_msg) from e + raise WeaviateConnectionError(self.__error_msg_with_detail(error_msg, e)) from e if isinstance(e, ReadTimeout): - raise WeaviateTimeoutError(error_msg) from e + raise WeaviateTimeoutError(self.__error_msg_with_detail(error_msg, e)) from e raise e + @staticmethod + def __error_msg_with_detail(error_msg: str, e: Exception) -> str: + detail = _exc_detail(e) + return f"{error_msg} ({detail})" if error_msg else detail + def __handle_response( self, response: Response, @@ -710,6 +794,7 @@ def exc(e: Exception) -> None: def close(self, colour: executor.Colour) -> executor.Result[None]: if self.embedded_db is not None: self.embedded_db.stop() + self._cancel_background_token_refresh() if colour == "async": async def execute() -> None: @@ -752,8 +837,11 @@ async def _execute() -> None: async with AsyncClient() as client: res = await client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) return resp(res) - except RequestError: - pass # ignore any errors related to requests, it is a best-effort warning + except (RequestError, OSError): + # ignore any errors related to requests, it is a best-effort warning. + # OSError covers fetch failures under Pyodide/WASM, where a page CSP + # commonly blocks pypi.org — that must not fail connect(). + pass return _execute() @@ -761,7 +849,7 @@ async def _execute() -> None: with Client() as client: res = client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) return resp(res) - except RequestError: + except (RequestError, OSError): pass # ignore any errors related to requests, it is a best-effort warning def delete( @@ -903,47 +991,49 @@ def connect(self, force: bool = False) -> None: self._open_connections_rest(self._auth, "sync") - # need this to get the version of weaviate for version checks and proper GRPC configuration try: - meta = executor.result(self.get_meta(False)) - self._weaviate_version = _ServerVersion.from_string(meta["version"]) - if "grpcMaxMessageSize" in meta: - self._grpc_max_msg_size = int(meta["grpcMaxMessageSize"]) - # Add warning later, when weaviate supported it for a while - # else: - # _Warnings.grpc_max_msg_size_not_found() - except ( - WeaviateConnectionError, - ReadError, - RemoteProtocolError, - SSLZeroReturnError, # required for async 3.8,3.9 due to ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1131) - ) as e: - self._connected = False - raise WeaviateStartUpError(f"Could not connect to Weaviate:{e}.") from e - - self.open_connection_grpc("sync") - if self.embedded_db is not None: + # need this to get the version of weaviate for version checks and proper GRPC configuration try: - self.wait_for_weaviate(10) - except WeaviateStartUpError as e: - self.embedded_db.stop() - self._connected = False - raise e - - # do it after all other init checks so as not to break all the tests - if self._weaviate_version.is_lower_than(1, 27, patch=0): - self._connected = False - raise WeaviateStartUpError( - f"Weaviate version {self._weaviate_version} is not supported. Please use Weaviate version 1.27.0 or higher." - ) + meta = executor.result(self.get_meta(False)) + self._weaviate_version = _ServerVersion.from_string(meta["version"]) + if "grpcMaxMessageSize" in meta: + self._grpc_max_msg_size = int(meta["grpcMaxMessageSize"]) + # Add warning later, when weaviate supported it for a while + # else: + # _Warnings.grpc_max_msg_size_not_found() + except ( + WeaviateConnectionError, + ReadError, + RemoteProtocolError, + SSLZeroReturnError, # required for async 3.8,3.9 due to ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1131) + ) as e: + raise WeaviateStartUpError( + f"Could not connect to Weaviate: {_exc_detail(e)}." + ) from e + + self.open_connection_grpc("sync") + if self.embedded_db is not None: + try: + self.wait_for_weaviate(10) + except WeaviateStartUpError: + self.embedded_db.stop() + raise + + # do it after all other init checks so as not to break all the tests + if self._weaviate_version.is_lower_than(1, 27, patch=0): + raise WeaviateStartUpError( + f"Weaviate version {self._weaviate_version} is not supported. Please use Weaviate version 1.27.0 or higher." + ) - if not self._skip_init_checks: - try: + if not self._skip_init_checks: executor.result(self._ping_grpc("sync")) executor.result(self._check_package_version("sync")) - except Exception as e: - self._connected = False - raise e + except BaseException: + # the OIDC step above may already have started the background token + # refresher; a failed connect must not leave it running + self._connected = False + self._cancel_background_token_refresh() + raise self._connected = True @@ -1107,47 +1197,50 @@ async def connect(self, force: bool = False) -> None: await executor.aresult(self._open_connections_rest(self._auth, "async")) - # need this to get the version of weaviate for version checks and proper GRPC configuration try: - meta = await self.get_meta(False) - self._weaviate_version = _ServerVersion.from_string(meta["version"]) - if "grpcMaxMessageSize" in meta: - self._grpc_max_msg_size = int(meta["grpcMaxMessageSize"]) - # Add warning later, when weaviate supported it for a while - # else: - # _Warnings.grpc_max_msg_size_not_found() - except ( - WeaviateConnectionError, - ReadError, - RemoteProtocolError, - SSLZeroReturnError, # required for async 3.8,3.9 due to ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1131) - ) as e: - self._connected = False - raise WeaviateStartUpError(f"Could not connect to Weaviate:{e}.") from e - - self.open_connection_grpc("async") - if self.embedded_db is not None: + # need this to get the version of weaviate for version checks and proper GRPC configuration try: - await self.wait_for_weaviate(10) - except WeaviateStartUpError as e: - self.embedded_db.stop() - self._connected = False - raise e - - # do it after all other init checks so as not to break all the tests - if self._weaviate_version.is_lower_than(1, 27, 0): - self._connected = False - raise WeaviateStartUpError( - f"Weaviate version {self._weaviate_version} is not supported. Please use Weaviate version 1.27.0 or higher." - ) + meta = await self.get_meta(False) + self._weaviate_version = _ServerVersion.from_string(meta["version"]) + if "grpcMaxMessageSize" in meta: + self._grpc_max_msg_size = int(meta["grpcMaxMessageSize"]) + # Add warning later, when weaviate supported it for a while + # else: + # _Warnings.grpc_max_msg_size_not_found() + except ( + WeaviateConnectionError, + ReadError, + RemoteProtocolError, + SSLZeroReturnError, # required for async 3.8,3.9 due to ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1131) + ) as e: + raise WeaviateStartUpError( + f"Could not connect to Weaviate: {_exc_detail(e)}." + ) from e + + self.open_connection_grpc("async") + if self.embedded_db is not None: + try: + await self.wait_for_weaviate(10) + except WeaviateStartUpError: + self.embedded_db.stop() + raise + + # do it after all other init checks so as not to break all the tests + if self._weaviate_version.is_lower_than(1, 27, 0): + raise WeaviateStartUpError( + f"Weaviate version {self._weaviate_version} is not supported. Please use Weaviate version 1.27.0 or higher." + ) - if not self._skip_init_checks: - try: + if not self._skip_init_checks: await executor.aresult(self._ping_grpc("async")) await executor.aresult(self._check_package_version("async")) - except Exception as e: - self._connected = False - raise e + except BaseException: + # the OIDC step above may already have started the background token + # refresher; a failed connect must not leave it running (it would keep + # hitting the IdP with no way for the user to stop it) + self._connected = False + self._cancel_background_token_refresh() + raise self._connected = True @@ -1159,7 +1252,9 @@ async def wait_for_weaviate(self, startup_period: int) -> None: ).raise_for_status() return except (ConnectError, ReadError, TimeoutError, HTTPStatusError): - time.sleep(1) + # asyncio.sleep, not time.sleep: a blocking sleep inside a coroutine + # stalls the event loop (and deadlocks single-threaded WASM runtimes) + await asyncio.sleep(1) try: ( diff --git a/weaviate/embedded.py b/weaviate/embedded.py index a511665cc..fb5a19a15 100644 --- a/weaviate/embedded.py +++ b/weaviate/embedded.py @@ -5,6 +5,7 @@ import socket import stat import subprocess +import sys import tarfile import time import urllib.request @@ -175,6 +176,14 @@ def wait_till_listening(self) -> None: @staticmethod def check_supported_platform() -> None: + if sys.platform == "emscripten": + # without this guard the port probe below "succeeds" under Emscripten's lazy + # socket emulation and misreports that Weaviate is already listening + raise WeaviateStartUpError( + "Embedded Weaviate is not supported under WebAssembly/Pyodide: it spawns a " + "local Weaviate subprocess, and processes are unavailable in the browser. " + "Connect to a remote Weaviate instance instead." + ) if platform.system() in ["Windows"]: raise WeaviateStartUpError( f"""{platform.system()} is not supported with EmbeddedDB. Please upvote this feature request if you want From 0d685a4cac7d5dbddd0b3d8b08dab2ed23feae38 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:20:09 +0300 Subject: [PATCH 08/27] fix(grpc-web): harden the fetch transport and error reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - map pyfetch failures into httpx's exception taxonomy (ConnectError / ReadTimeout) so the base client classifies them and best-effort callers that swallow RequestError keep working — previously every network/CORS/CSP/abort failure surfaced as a raw OSError - defer to Pyodide's distributed httpx when its native jsfetch transport is present (>= 0.27 dist builds): it streams, splits connect/read timeouts, and raises real httpx errors, so overwriting it made things worse; the pyfetch transport remains the fallback for PyPI httpx - strip content-encoding/content-length from responses: fetch hands back already-decompressed bytes and httpx would gunzip them a second time and raise DecodingError (hidden live only because CORS masks the header cross-origin) - pick the first non-None timeout instead of an or-chain (an explicit read=0 no longer falls through to the 5s connect value), fail fast at install when pyodide is missing, add uninstall_fetch_transport() and a sentinel on the patched method, and restore CRLF header validation that bypassing h11 had lost - channel: name exception types in transport errors (str() of httpx errors can be empty), hint at Access-Control-Expose-Headers when a trailers-only response lost its grpc-status to CORS, and stop recommending batch.dynamic()/fixed_size()/rate_limit(), which do not exist on the async client - README: correct the support table and document CORS requirements, browser-ignored configuration, OIDC, embedded, and agents support - add a dedicated test suite for the fetch transport (in-process fakes + subprocess installs) Co-Authored-By: Claude Fable 5 --- packages/grpc-web/README.md | 51 +- .../src/weaviate_grpc_web/__init__.py | 2 + .../src/weaviate_grpc_web/_channel.py | 28 +- .../src/weaviate_grpc_web/_httpx_fetch.py | 143 ++++- packages/grpc-web/tests/test_httpx_fetch.py | 503 ++++++++++++++++++ packages/grpc-web/tests/test_transport.py | 47 ++ 6 files changed, 748 insertions(+), 26 deletions(-) create mode 100644 packages/grpc-web/tests/test_httpx_fetch.py diff --git a/packages/grpc-web/README.md b/packages/grpc-web/README.md index 5810e9f06..c63e10bd1 100644 --- a/packages/grpc-web/README.md +++ b/packages/grpc-web/README.md @@ -27,6 +27,12 @@ and POSTs them via `pyodide.http.pyfetch` to a server fronted by a grpc-web tran (e.g. Envoy or [connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)). Call metadata (API key / OIDC bearer) is folded into `fetch` headers. +For REST, Pyodide ≥ 0.27 distributes a patched httpx that already routes through the +browser's `fetch` natively — when that build is detected the package leaves it alone. +Only when httpx resolved from PyPI (httpcore + raw sockets, which cannot work under +WASM) does the package patch `httpx.AsyncHTTPTransport` with its own pyfetch-based +transport. + ## Usage ```python @@ -41,13 +47,48 @@ await collection.query.near_text("hello", limit=3) ## Supported / unsupported -| RPC | Kind | Status | +| Feature | Kind | Status | |----------------------------------------------------------|-----------------|--------| -| Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary | ✅ works over grpc-web | -| Health check (`/grpc.health.v1.Health/Check`) | unary | ✅ (recommend `skip_init_checks=True` + REST `/.well-known/ready`) | -| References (`/batch/references`) | REST | ✅ via httpx-in-Pyodide | -| `batch.stream()` / `batch.experimental()` (BatchStream) | bidi streaming | ❌ not possible over grpc-web/fetch — use `insert_many()` / `batch.dynamic()` / `fixed_size()` / `rate_limit()` | +| Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary gRPC | ✅ works over grpc-web | +| Health check (`/grpc.health.v1.Health/Check`) | unary gRPC | ✅ (recommend `skip_init_checks=True` + REST `/.well-known/ready`) | +| REST (`is_ready`, config, `/batch/references`, …) | REST | ✅ via fetch (Pyodide's httpx build, or this package's fallback transport) | +| API-key auth (`Auth.api_key`) | header | ✅ | +| OIDC auth (`client_credentials` / `client_password` / `bearer_token`) | REST | ✅ token fetch + asyncio-task refresh (no threads) | +| Bulk insert: `collection.data.insert_many()` | unary gRPC | ✅ the supported bulk path under WASM | +| `batch.stream()` / `batch.experimental()` (BatchStream) | bidi streaming | ❌ not possible over grpc-web/fetch — raises immediately; use `insert_many()` | +| `batch.dynamic()` / `fixed_size()` / `rate_limit()` | sync-client API | ❌ these only exist on the sync client, which is unsupported under WASM | +| Embedded Weaviate (`use_async_with_embedded`) | subprocess | ❌ raises "not supported under WebAssembly/Pyodide" | | Synchronous client | — | ❌ async-only under WASM | +| Weaviate Agents: `AsyncQueryAgent` `run/ask/search` | REST | ✅ via fetch | +| Weaviate Agents: `ask_stream` / `research_stream` (SSE) | REST streaming | ⚠️ degraded under the fallback transport: fully buffered, events arrive only when the run completes (and long runs can hit the request timeout) | +| Weaviate Agents: sync `QueryAgent`, `TransformationAgent`, `PersonalizationAgent` | REST sync | ❌ no async flavour exists | + +## Configuration not honored in the browser + +`fetch` manages connections itself, so several knobs are accepted but have no effect +under WASM: + +- `AdditionalConfig.proxies` / `trust_env` proxy environment variables (the browser + cannot proxy fetch requests per-client), +- connection-pool sizing and `session_pool_max_retries`, +- `GrpcConfig.credentials` (custom CA bundles — the browser's trust store decides TLS), +- `GrpcConfig.channel_options`, including `grpc.max_send/receive_message_length` + (only `grpc-web.path_prefix` is consumed), +- `Proxies.grpc` / `GRPC_PROXY`. + +## CORS requirements (browsers) + +Cross-origin browser deployments must configure the grpc-web transcoder / REST endpoint +with CORS, or failures become hard to diagnose: + +- allow the request headers the client sends: `authorization`, `content-type`, + `x-grpc-web`, and any custom headers; +- expose the grpc-web status headers on responses: + `Access-Control-Expose-Headers: grpc-status, grpc-message` — without this, + trailers-only error responses (e.g. a bad API key) are reported as + `INTERNAL: grpc-web response contained no message frame` instead of the real error; +- note that a CORS-blocked request is indistinguishable from a network failure in the + browser (`TypeError: Failed to fetch`), and is retried as UNAVAILABLE. ## Testing on CPython diff --git a/packages/grpc-web/src/weaviate_grpc_web/__init__.py b/packages/grpc-web/src/weaviate_grpc_web/__init__.py index 542d075bd..ba9fa14b3 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/__init__.py +++ b/packages/grpc-web/src/weaviate_grpc_web/__init__.py @@ -27,6 +27,7 @@ "install", "is_installed", "install_fetch_transport", + "uninstall_fetch_transport", "is_fetch_transport_installed", "set_sender", "make_httpx_sender", @@ -58,5 +59,6 @@ def _bootstrap() -> None: from ._httpx_fetch import ( # noqa: E402 install_fetch_transport, is_fetch_transport_installed, + uninstall_fetch_transport, ) from ._sender import make_httpx_sender # noqa: E402 diff --git a/packages/grpc-web/src/weaviate_grpc_web/_channel.py b/packages/grpc-web/src/weaviate_grpc_web/_channel.py index cb16f7bc2..e2dcb4dd7 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_channel.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_channel.py @@ -114,10 +114,13 @@ def __init__(self, path: str) -> None: self._path = path def __call__(self, *args: Any, **kwargs: Any) -> Any: + # NOTE: do not recommend batch.dynamic()/fixed_size()/rate_limit() here — those + # are sync-client-only APIs and do not exist on the async client, which is the + # only client supported under WASM. raise RuntimeError( f"Bidirectional streaming RPC {self._path!r} (server-side batching / " - "BatchStream) is not supported over grpc-web/fetch. Use insert_many(), or " - "batch.dynamic() / fixed_size() / rate_limit(), instead of batch.stream()." + "BatchStream) is not supported over grpc-web/fetch. Use " + "collection.data.insert_many() instead of batch.stream()." ) @@ -202,9 +205,12 @@ async def _unary( details=f"grpc-web request to {path} timed out after {timeout}s", ) from exc except Exception as exc: # network/transport failure -> retryable UNAVAILABLE + # str() of transport errors can be empty (e.g. httpx.ConnectError) — always + # include the exception type so failures stay diagnosable + detail = f"{type(exc).__name__}: {exc}" if str(exc) else repr(exc) raise AioRpcError( code=StatusCode.UNAVAILABLE, - details=f"grpc-web transport error for {path}: {exc}", + details=f"grpc-web transport error for {path}: {detail}", ) from exc try: @@ -247,10 +253,18 @@ def _handle_response( if code is not StatusCode.OK: raise AioRpcError(code=code, details=message) if not messages: - raise AioRpcError( - code=StatusCode.INTERNAL, - details="grpc-web response contained no message frame", - ) + details = "grpc-web response contained no message frame" + if raw_status is None: + # HTTP 200, no body frames, and no grpc-status anywhere: the classic + # signature of a trailers-only error response whose grpc-status / + # grpc-message headers were stripped by CORS in the browser. + details += ( + " and no grpc-status was visible. If this is a cross-origin browser " + "request, configure the grpc-web proxy to send " + "'Access-Control-Expose-Headers: grpc-status, grpc-message' so " + "trailers-only error responses are readable." + ) + raise AioRpcError(code=StatusCode.INTERNAL, details=details) return deserialize(messages[0]) diff --git a/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py b/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py index 8a84e5d11..f09d6bfbe 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py @@ -10,14 +10,27 @@ ``fetch`` via ``pyodide.http.pyfetch`` — the same install-globally-under-Emscripten philosophy as the grpc shim in ``_shim.py``. Responses are fully buffered, which matches how the base client consumes them (JSON bodies, no streaming). + +NOTE: Pyodide >= 0.27 distributes a patched httpx whose ``AsyncHTTPTransport`` already +routes through JS ``fetch`` natively (``httpx/_transports/jsfetch.py``) with streaming +support and a proper connect/read timeout split. When that build is detected, installing +is skipped — overwriting it would replace a better implementation. This transport is the +fallback for environments where httpx resolved from PyPI (httpcore + raw sockets). + +Known divergences from native httpx (acceptable for the weaviate client's usage): +- the browser's fetch follows redirects internally, so httpx never sees a 3xx; +- multi-value response headers (e.g. Set-Cookie) are folded into one value; +- responses are fully buffered (no streaming). """ +import importlib.util import sys -from typing import Dict +from typing import Callable, Dict, Optional import httpx _installed = False +_original_handle_async_request: Optional[Callable] = None # Hop-by-hop / connection-managed headers that the browser's fetch controls itself. # Browsers silently drop forbidden headers, but Node's undici (used by the CPython/Node @@ -30,6 +43,19 @@ "transfer-encoding", } +# Response headers describing the wire encoding of the body. fetch decompresses +# responses transparently, so the bytes handed to httpx are already plain; passing the +# original content-encoding through makes httpx run its decoders over them again and +# raise DecodingError, and the original content-length no longer matches the body. +# (Browsers usually hide content-encoding on CORS responses, which is why this never +# fired live — same-origin and Node fetch do expose it.) +_FETCH_DECODED_RESPONSE_HEADERS = { + "content-encoding", + "content-length", +} + +_TIMEOUT_HINTS = ("timeout", "timed out", "abort") + async def _read_request_body(request: httpx.Request) -> bytes: try: @@ -38,34 +64,86 @@ async def _read_request_body(request: httpx.Request) -> bytes: return await request.aread() +def _pick_timeout(request: httpx.Request) -> Optional[float]: + """Pick the effective deadline from httpx's timeout extension. + + httpx populates ``extensions['timeout']`` with connect/read/write/pool values; the + read timeout is what the weaviate client configures per request. ``get(...) or`` + chains would silently skip an explicit 0, so check for None instead. + """ + timeouts = request.extensions.get("timeout") or {} + for key in ("read", "connect", "pool"): + value = timeouts.get(key) + if value is not None: + return value + return None + + +def _map_fetch_error( + e: BaseException, request: httpx.Request, deadline_set: bool +) -> httpx.TransportError: + """Translate a pyfetch failure into httpx's exception taxonomy. + + Pyodide surfaces every JS fetch rejection (network down, DNS, CORS, CSP, an + AbortSignal firing) as OSError — or pyodide.http.AbortError, an OSError subclass — + never as an httpx exception. Without this mapping the base client cannot classify + failures (WeaviateConnectionError/WeaviateTimeoutError) and best-effort callers that + swallow httpx.RequestError break. + """ + msg = str(e) or repr(e) + if deadline_set and any(hint in msg.lower() for hint in _TIMEOUT_HINTS): + return httpx.ReadTimeout(msg, request=request) + return httpx.ConnectError(msg, request=request) + + +def _validate_header(name: str, value: str) -> None: + # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at + # send time; this transport bypasses h11, so mirror that defence here rather than + # delegating it entirely to the JS runtime's fetch. + if any(c in name or c in value for c in ("\r", "\n", "\0")): + raise httpx.LocalProtocolError(f"Illegal character in header {name!r}") + + async def _fetch_handle_async_request( self: httpx.AsyncHTTPTransport, request: httpx.Request ) -> httpx.Response: from pyodide.http import pyfetch # type: ignore[import-not-found] - headers: Dict[str, str] = { - k: v for k, v in request.headers.items() if k.lower() not in _FETCH_MANAGED_HEADERS - } + headers: Dict[str, str] = {} + for k, v in request.headers.items(): + if k.lower() in _FETCH_MANAGED_HEADERS: + continue + _validate_header(k, v) + headers[k] = v kwargs: Dict[str, object] = {} body = await _read_request_body(request) if body: # fetch rejects GET/HEAD requests that carry a body kwargs["body"] = body - timeouts = request.extensions.get("timeout") or {} - timeout = timeouts.get("read") or timeouts.get("connect") or timeouts.get("pool") - if timeout: + timeout = _pick_timeout(request) + deadline_set = False + if timeout is not None and timeout > 0: try: from js import AbortSignal # type: ignore[import-not-found] kwargs["signal"] = AbortSignal.timeout(int(timeout * 1000)) + deadline_set = True except Exception: # pragma: no cover - AbortSignal.timeout availability varies pass - response = await pyfetch(str(request.url), method=request.method, headers=headers, **kwargs) - data = await response.bytes() try: - resp_headers = dict(response.headers) + response = await pyfetch(str(request.url), method=request.method, headers=headers, **kwargs) + data = await response.bytes() + except OSError as e: # incl. pyodide.http.AbortError + raise _map_fetch_error(e, request, deadline_set) from e + + try: + resp_headers = { + k: v + for k, v in dict(response.headers).items() + if k.lower() not in _FETCH_DECODED_RESPONSE_HEADERS + } except Exception: # pragma: no cover - header shape varies across Pyodide versions resp_headers = {} return httpx.Response( @@ -76,20 +154,57 @@ async def _fetch_handle_async_request( ) +# sentinel so other packages (and uninstall) can recognise the patched method +_fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] + + +def _platform_httpx_has_fetch_support() -> bool: + """True when the running httpx is Pyodide's distributed build. + + That build replaces the httpcore transport with a native JS-fetch one + (httpx/_transports/jsfetch.py), so the weaviate REST path already works without + this shim — and works better (streaming, connect/read timeout split). + """ + try: + return importlib.util.find_spec("httpx._transports.jsfetch") is not None + except (ImportError, ValueError): # pragma: no cover - exotic import states + return False + + def install_fetch_transport(force: bool = False) -> None: """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. Installs only under Emscripten unless ``force=True`` (CPython testing, where a - ``pyodide`` stub must be importable). Idempotent. + ``pyodide`` stub must be importable), and is skipped when httpx itself already has + fetch support (Pyodide's distributed build). Idempotent. """ - global _installed + global _installed, _original_handle_async_request if _installed: return - if not force and sys.platform != "emscripten": - return + if not force: + if sys.platform != "emscripten": + return + if _platform_httpx_has_fetch_support(): + return + # Fail fast: the handler imports pyfetch per request, so a missing pyodide module + # would otherwise surface as a confusing ModuleNotFoundError on the first request. + from pyodide.http import pyfetch # type: ignore[import-not-found] # noqa: F401 + + _original_handle_async_request = httpx.AsyncHTTPTransport.handle_async_request httpx.AsyncHTTPTransport.handle_async_request = _fetch_handle_async_request # type: ignore[method-assign] _installed = True +def uninstall_fetch_transport() -> None: + """Restore the original ``httpx.AsyncHTTPTransport`` behaviour. No-op if not installed.""" + global _installed, _original_handle_async_request + if not _installed: + return + assert _original_handle_async_request is not None + httpx.AsyncHTTPTransport.handle_async_request = _original_handle_async_request # type: ignore[method-assign] + _original_handle_async_request = None + _installed = False + + def is_fetch_transport_installed() -> bool: return _installed diff --git a/packages/grpc-web/tests/test_httpx_fetch.py b/packages/grpc-web/tests/test_httpx_fetch.py new file mode 100644 index 000000000..f6c866100 --- /dev/null +++ b/packages/grpc-web/tests/test_httpx_fetch.py @@ -0,0 +1,503 @@ +"""Tests for the fetch-based httpx transport (_httpx_fetch.py). + +In-process tests call ``_fetch_handle_async_request`` directly with a fake +``pyodide.http`` module injected into ``sys.modules`` — no global monkeypatch of +``httpx.AsyncHTTPTransport`` is needed, so the real httpx in the dev environment is left +untouched. Install semantics (which DO patch the class globally) run in fresh +subprocesses, mirroring test_shim_install.py. +""" + +import asyncio +import pathlib +import subprocess +import sys +import textwrap +import types +from typing import Any, Dict, List, Optional + +import httpx +import pytest + +from weaviate_grpc_web._httpx_fetch import _fetch_handle_async_request + +_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") + + +class FakeFetchResponse: + def __init__(self, status: int = 200, headers: Optional[Any] = None, body: bytes = b""): + self.status = status + self.headers: Any = headers or {} + self._body = body + + async def bytes(self) -> bytes: # noqa: A003 - mirrors pyodide's FetchResponse API + return self._body + + +class FakePyfetch: + def __init__(self, response: Optional[FakeFetchResponse] = None): + self.response = response or FakeFetchResponse() + self.calls: List[Dict[str, Any]] = [] + + async def __call__(self, url: str, **kwargs: Any) -> FakeFetchResponse: + self.calls.append({"url": url, **kwargs}) + return self.response + + +@pytest.fixture +def fake_pyfetch(monkeypatch) -> FakePyfetch: + fetch = FakePyfetch() + pyodide_mod = types.ModuleType("pyodide") + http_mod = types.ModuleType("pyodide.http") + http_mod.pyfetch = fetch # type: ignore[attr-defined] + pyodide_mod.http = http_mod # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) + monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) + return fetch + + +def _handle(request: httpx.Request) -> httpx.Response: + # self is unused by the handler implementation; a bare transport instance suffices + transport = httpx.AsyncHTTPTransport.__new__(httpx.AsyncHTTPTransport) + return asyncio.run(_fetch_handle_async_request(transport, request)) + + +def test_basic_get_round_trip(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse( + status=200, headers={"content-type": "application/json"}, body=b'{"version": "1.30.0"}' + ) + response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + + assert response.status_code == 200 + assert response.json() == {"version": "1.30.0"} + assert response.headers["content-type"] == "application/json" + call = fake_pyfetch.calls[0] + assert call["url"] == "http://h:8080/v1/meta" + assert call["method"] == "GET" + + +def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): + fake_pyfetch.response = FakeFetchResponse(status=404, body=b"") + response = _handle(httpx.Request("GET", "http://h:8080/v1/schema/Nope")) + with pytest.raises(httpx.HTTPStatusError): + response.raise_for_status() + + +def test_fetch_managed_request_headers_stripped(fake_pyfetch): + request = httpx.Request( + "POST", + "http://h:8080/v1/objects", + headers={ + "authorization": "Bearer k", + "content-type": "application/json", + "host": "h:8080", + "connection": "keep-alive", + "accept-encoding": "gzip", + "transfer-encoding": "chunked", + }, + content=b"{}", + ) + _handle(request) + sent = fake_pyfetch.calls[0]["headers"] + assert sent["authorization"] == "Bearer k" + assert sent["content-type"] == "application/json" + for managed in ("host", "connection", "accept-encoding", "content-length", "transfer-encoding"): + assert managed not in sent + + +def test_get_without_body_omits_body_kwarg(fake_pyfetch): + # fetch rejects GET/HEAD requests that carry a body, so the kwarg must be absent + _handle(httpx.Request("GET", "http://h:8080/v1/.well-known/ready")) + assert "body" not in fake_pyfetch.calls[0] + + +def test_post_body_passed(fake_pyfetch): + _handle(httpx.Request("POST", "http://h:8080/v1/graphql", content=b'{"query": "x"}')) + assert fake_pyfetch.calls[0]["body"] == b'{"query": "x"}' + + +def test_delete_with_body_passed(fake_pyfetch): + # the REST batch-delete path sends DELETE with a JSON body + _handle(httpx.Request("DELETE", "http://h:8080/v1/batch/objects", content=b'{"match": {}}')) + assert fake_pyfetch.calls[0]["body"] == b'{"match": {}}' + + +def test_query_string_preserved_in_url(fake_pyfetch): + _handle(httpx.Request("GET", "http://h:8080/v1/objects?class=A&limit=10&after=a%20b")) + assert fake_pyfetch.calls[0]["url"] == "http://h:8080/v1/objects?class=A&limit=10&after=a%20b" + + +def test_content_encoding_stripped_from_response(fake_pyfetch): + # fetch hands back ALREADY-decompressed bytes; if the original content-encoding + # header were passed through, httpx.Response would gunzip a second time and raise + # DecodingError. content-length is stale for the same reason. + fake_pyfetch.response = FakeFetchResponse( + status=200, + headers={"content-encoding": "gzip", "content-length": "23", "x-other": "kept"}, + body=b'{"version": "1.30.0"}', + ) + response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.json() == {"version": "1.30.0"} + assert "content-encoding" not in response.headers + assert response.headers["x-other"] == "kept" + + +def test_unreadable_response_headers_tolerated(fake_pyfetch): + class BadHeaders: + def keys(self): + raise TypeError("header shape varies across Pyodide versions") + + fake_pyfetch.response = FakeFetchResponse(status=200, body=b"ok") + fake_pyfetch.response.headers = BadHeaders() + response = _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert response.status_code == 200 + assert response.content == b"ok" + + +class _AbortSignalRecorder: + def __init__(self): + self.timeouts: List[int] = [] + + def timeout(self, ms: int): + self.timeouts.append(ms) + return f"signal-{ms}" + + +@pytest.fixture +def fake_abort_signal(monkeypatch) -> _AbortSignalRecorder: + recorder = _AbortSignalRecorder() + js_mod = types.ModuleType("js") + js_mod.AbortSignal = recorder # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "js", js_mod) + return recorder + + +def _request_with_timeout(timeouts: Dict[str, Optional[float]]) -> httpx.Request: + request = httpx.Request("GET", "http://h:8080/v1/meta") + request.extensions["timeout"] = timeouts + return request + + +def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): + # mirrors what weaviate's AsyncClient puts in extensions: connect/read/write/pool + _handle(_request_with_timeout({"connect": 2.0, "read": 30.0, "write": 5.0, "pool": 9.0})) + assert fake_abort_signal.timeouts == [30000] + assert fake_pyfetch.calls[0]["signal"] == "signal-30000" + + +def test_timeout_falls_back_to_connect_then_pool(fake_pyfetch, fake_abort_signal): + _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) + _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 9.0})) + assert fake_abort_signal.timeouts == [2000, 9000] + assert [c["signal"] for c in fake_pyfetch.calls] == ["signal-2000", "signal-9000"] + + +def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): + _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +def test_missing_js_module_degrades_to_no_signal(fake_pyfetch): + # off-browser (no js module) the AbortSignal import fails; the request must still go out + assert "js" not in sys.modules + response = _handle( + _request_with_timeout({"connect": 2.0, "read": 30.0, "write": None, "pool": None}) + ) + assert response.status_code == 200 + assert "signal" not in fake_pyfetch.calls[0] + + +def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): + # an explicit read=0 must not fall through to the 5s connect timeout, nor become an + # immediate AbortSignal.timeout(0) + _handle(_request_with_timeout({"connect": 5.0, "read": 0, "write": None, "pool": None})) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +class RaisingPyfetch: + def __init__(self, exc: BaseException): + self.exc = exc + + async def __call__(self, url: str, **kwargs: Any): + raise self.exc + + +def _install_raising_pyfetch(monkeypatch, exc: BaseException) -> None: + pyodide_mod = types.ModuleType("pyodide") + http_mod = types.ModuleType("pyodide.http") + http_mod.pyfetch = RaisingPyfetch(exc) # type: ignore[attr-defined] + pyodide_mod.http = http_mod # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "pyodide", pyodide_mod) + monkeypatch.setitem(sys.modules, "pyodide.http", http_mod) + + +def test_fetch_failure_maps_to_httpx_connect_error(monkeypatch): + # pyodide surfaces JS fetch rejections as OSError; the base client can only classify + # httpx exceptions (WeaviateConnectionError etc.), so the shim must translate + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch") as excinfo: + _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert isinstance(excinfo.value.__cause__, OSError) + + +def test_fetch_abort_with_deadline_maps_to_read_timeout(monkeypatch, fake_abort_signal): + # AbortSignal.timeout firing surfaces as an OSError subclass mentioning the abort; + # with a deadline set this must classify as a timeout, not a connection error + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + with pytest.raises(httpx.ReadTimeout, match="signal timed out"): + _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + + +def test_fetch_failure_with_deadline_but_no_timeout_message_stays_connect_error( + monkeypatch, fake_abort_signal +): + # nearly every weaviate request sets a read deadline; a plain network failure on + # such a request must remain a connection error, not become a timeout + _install_raising_pyfetch(monkeypatch, OSError("TypeError: Failed to fetch")) + with pytest.raises(httpx.ConnectError, match="Failed to fetch"): + _handle(_request_with_timeout({"connect": None, "read": 30.0, "write": None, "pool": None})) + + +def test_fetch_abort_without_deadline_stays_connect_error(monkeypatch): + # the same message without a deadline set (no js module -> no signal) is not OUR + # timeout, so it must stay a connection error + _install_raising_pyfetch(monkeypatch, OSError("AbortError: signal timed out")) + assert "js" not in sys.modules + with pytest.raises(httpx.ConnectError): + _handle(_request_with_timeout({"connect": None, "read": 0.5, "write": None, "pool": None})) + + +def test_empty_oserror_str_keeps_repr_detail(monkeypatch): + _install_raising_pyfetch(monkeypatch, OSError()) + with pytest.raises(httpx.ConnectError) as excinfo: + _handle(httpx.Request("GET", "http://h:8080/v1/meta")) + assert "OSError" in str(excinfo.value) + + +def test_crlf_in_header_value_rejected(fake_pyfetch): + # httpx.Request accepts CR/LF in header values and relies on h11 to reject them at + # send time; this transport bypasses h11 and must keep that defence + request = httpx.Request( + "GET", "http://h:8080/v1/meta", headers={"x-key": "val\r\nx-injected: evil"} + ) + with pytest.raises(httpx.LocalProtocolError): + _handle(request) + assert fake_pyfetch.calls == [] + + +def test_platform_jsfetch_detection(monkeypatch): + import importlib.machinery + + from weaviate_grpc_web._httpx_fetch import _platform_httpx_has_fetch_support + + # the dev environment runs PyPI httpx (httpcore-based): no jsfetch transport + assert _platform_httpx_has_fetch_support() is False + + fake = types.ModuleType("httpx._transports.jsfetch") + fake.__spec__ = importlib.machinery.ModuleSpec("httpx._transports.jsfetch", loader=None) + monkeypatch.setitem(sys.modules, "httpx._transports.jsfetch", fake) + assert _platform_httpx_has_fetch_support() is True + + +# --------------------------------------------------------------------------- +# Install semantics: these patch httpx.AsyncHTTPTransport globally, so each +# scenario runs in a fresh subprocess (same pattern as test_shim_install.py). +# --------------------------------------------------------------------------- + +_FAKE_PYODIDE_PRELUDE = """ +import sys, types + +class _FakeResponse: + status = 200 + headers = {"content-type": "application/json"} + async def bytes(self): + return b'{"ok": true}' + +CALLS = [] +async def pyfetch(url, **kwargs): + CALLS.append((url, kwargs)) + return _FakeResponse() + +_pyodide = types.ModuleType("pyodide") +_http = types.ModuleType("pyodide.http") +_http.pyfetch = pyfetch +_pyodide.http = _http +sys.modules["pyodide"] = _pyodide +sys.modules["pyodide.http"] = _http +""" + + +def _run(body: str, prelude: str = "") -> subprocess.CompletedProcess: + script = f"import sys\nsys.path.insert(0, {_SRC!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + +def test_force_install_routes_async_client_through_pyfetch(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import asyncio, httpx + from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + + install_fetch_transport(force=True) + assert is_fetch_transport_installed() + + async def main(): + async with httpx.AsyncClient() as client: + return await client.get("http://h:8080/v1/meta") + + resp = asyncio.run(main()) + assert resp.status_code == 200, resp.status_code + assert resp.json() == {"ok": True} + assert CALLS and CALLS[0][0] == "http://h:8080/v1/meta" + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_install_without_force_is_noop_off_emscripten(): + result = _run( + """ + import sys + assert sys.platform != "emscripten" + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + install_fetch_transport() + assert not is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is before + print("OK") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_force_install_is_idempotent(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + from weaviate_grpc_web import install_fetch_transport + install_fetch_transport(force=True) + patched = httpx.AsyncHTTPTransport.handle_async_request + install_fetch_transport(force=True) + assert httpx.AsyncHTTPTransport.handle_async_request is patched + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_sync_transport_left_untouched(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + sync_before = httpx.HTTPTransport.handle_request + from weaviate_grpc_web import install_fetch_transport + install_fetch_transport(force=True) + assert httpx.HTTPTransport.handle_request is sync_before + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_uninstall_restores_original_transport(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_grpc_web import ( + install_fetch_transport, + is_fetch_transport_installed, + uninstall_fetch_transport, + ) + uninstall_fetch_transport() # no-op when not installed + install_fetch_transport(force=True) + assert is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is not before + uninstall_fetch_transport() + assert not is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is before + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_patched_method_carries_sentinel(): + result = _run( + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" + import httpx + from weaviate_grpc_web import install_fetch_transport + assert not getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ) + install_fetch_transport(force=True) + assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ) is True + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_force_install_without_pyodide_fails_fast(): + # without a pyodide module the install must raise immediately, not let every later + # request die with a lazy ModuleNotFoundError + result = _run( + """ + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + try: + install_fetch_transport(force=True) + except ModuleNotFoundError: + assert not is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is before + print("OK") + else: + raise AssertionError("expected install to fail fast without pyodide") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_emscripten_with_platform_jsfetch_skips_install(): + # on Pyodide's distributed httpx (jsfetch transport built in), the shim must NOT + # overwrite the platform implementation + result = _run( + """ + import importlib.machinery, sys, types + + sys.platform = "emscripten" + fake = types.ModuleType("httpx._transports.jsfetch") + fake.__spec__ = importlib.machinery.ModuleSpec( + "httpx._transports.jsfetch", loader=None + ) + sys.modules["httpx._transports.jsfetch"] = fake + + import httpx + before = httpx.AsyncHTTPTransport.handle_async_request + from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + install_fetch_transport() # no force: platform transport must win + assert not is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is before + print("OK") + """ + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/packages/grpc-web/tests/test_transport.py b/packages/grpc-web/tests/test_transport.py index f931fa8e1..bcbee6ed7 100644 --- a/packages/grpc-web/tests/test_transport.py +++ b/packages/grpc-web/tests/test_transport.py @@ -160,6 +160,53 @@ async def boom(url, headers, body, timeout): with pytest.raises(AioRpcError) as excinfo: asyncio.run(mc(b"q")) assert excinfo.value.code() is StatusCode.UNAVAILABLE + assert "ConnectionError: connection refused" in str(excinfo.value.details()) + + +def test_transport_exception_with_empty_str_keeps_type(): + # httpx transport errors commonly stringify to '' — the detail must still name them + async def boom(url, headers, body, timeout): + raise ConnectionError() + + channel = GrpcWebChannel("h:1", secure=False, sender=boom) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert "ConnectionError" in str(excinfo.value.details()) + + +def test_empty_ok_response_hints_at_cors_expose_headers(): + # HTTP 200, empty body, no grpc-status anywhere: the shape of a trailers-only error + # whose grpc-status/grpc-message headers were stripped by CORS + channel = _channel(FakeSender(status=200, headers={}, body=b"")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + assert "Access-Control-Expose-Headers" in str(excinfo.value.details()) + + +def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): + # when grpc-status WAS visible (status 0, no frames), it is a malformed response, + # not a CORS problem — the hint must not appear + channel = _channel(FakeSender(status=200, headers={"grpc-status": "0"}, body=b"")) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) + + +def test_stream_stream_error_recommends_insert_many_only(): + # batch.dynamic()/fixed_size()/rate_limit() do not exist on the async client (the + # only one supported under WASM), so the error must not recommend them + channel = _channel(FakeSender()) + mc = channel.stream_stream("/weaviate.v1.Weaviate/BatchStream", lambda x: x, lambda b: b) + with pytest.raises(RuntimeError) as excinfo: + mc(request_iterator=iter([]), timeout=5, metadata=None) + assert "insert_many" in str(excinfo.value) + for sync_only in ("dynamic", "fixed_size", "rate_limit"): + assert sync_only not in str(excinfo.value) def test_malformed_frame_maps_to_internal(): From c8f321ac4fb888dfb6e6a88c523516694e7dabed Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:20:27 +0300 Subject: [PATCH 09/27] chore: gitignore all egg-info dirs The literal weaviate_client.egg-info pattern missed the new packages/grpc-web build artifact; generalize it. Co-Authored-By: Claude Fable 5 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b4ba50e1b..bc3a48c66 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ venv .idea dist/ -weaviate_client.egg-info +*.egg-info/ **/__pycache__ tmp build/ From 6797d7478c8a1185c465dd222597d2a571b46218 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:29:11 +0200 Subject: [PATCH 10/27] ci: run and lint packages/grpc-web in the main workflow --- .github/workflows/main.yaml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 489ff9504..e2004af1c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -45,9 +45,9 @@ jobs: cache: 'pip' # caching pip dependencies - run: pip install -r requirements-devel.txt - name: "Ruff lint" - run: ruff check weaviate test mock_tests integration + run: ruff check weaviate test mock_tests integration packages/grpc-web - name: "Ruff format" - run: ruff format --diff weaviate test mock_tests integration + run: ruff format --diff weaviate test mock_tests integration packages/grpc-web - name: "Flake 8" run: flake8 weaviate test mock_tests integration - name: "Check release for pypi" @@ -105,6 +105,26 @@ jobs: name: coverage-report-${{ matrix.folder }} path: coverage-${{ matrix.folder }}.xml + grpc-web-tests: + name: Run gRPC-Web Package Tests + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ matrix.version }} + cache: 'pip' # caching pip dependencies + - run: | + pip install -r requirements-test.txt -r requirements-devel.txt + pip install -e . -e packages/grpc-web + - name: Run grpc-web package tests + run: pytest packages/grpc-web/tests + proto-test: name: Run importing protos test runs-on: ubuntu-latest From 6367217e07b0390e3c896240ca515fb681d818a5 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:30:02 +0200 Subject: [PATCH 11/27] fix(grpc-web): treat a missing grpc-status trailer as INTERNAL, not success Per the grpc-web contract every unary response must carry a grpc-status (trailer frame or header). A proxy that dropped the trailer frame previously read as OK and returned the first message frame. --- .../src/weaviate_grpc_web/_channel.py | 7 +++++++ packages/grpc-web/tests/test_transport.py | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/packages/grpc-web/src/weaviate_grpc_web/_channel.py b/packages/grpc-web/src/weaviate_grpc_web/_channel.py index e2dcb4dd7..aef41241f 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_channel.py +++ b/packages/grpc-web/src/weaviate_grpc_web/_channel.py @@ -246,6 +246,13 @@ def _handle_response( code=_status_from_http(http_status), details=f"HTTP {http_status} from grpc-web endpoint", ) + if messages: + # Every grpc-web unary response must carry a grpc-status (trailer frame + # or header); a proxy that drops the trailer must not read as success. + raise AioRpcError( + code=StatusCode.INTERNAL, + details="grpc-web response missing grpc-status trailers", + ) code = StatusCode.OK else: code = status_from_int(int(raw_status)) diff --git a/packages/grpc-web/tests/test_transport.py b/packages/grpc-web/tests/test_transport.py index bcbee6ed7..90268c02a 100644 --- a/packages/grpc-web/tests/test_transport.py +++ b/packages/grpc-web/tests/test_transport.py @@ -197,6 +197,27 @@ def test_empty_ok_response_with_grpc_status_has_no_cors_hint(): assert "Access-Control-Expose-Headers" not in str(excinfo.value.details()) +def test_message_frame_without_grpc_status_is_internal_not_success(): + # HTTP 200 with a valid message frame but no grpc-status anywhere (e.g. a proxy + # dropped the trailer frame) must be an error, never a fabricated success + channel = _channel(FakeSender(status=200, headers={}, body=_frame(b"reply-bytes"))) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.INTERNAL + assert "missing grpc-status" in str(excinfo.value.details()) + + +def test_message_frame_with_grpc_status_header_still_succeeds(): + # trailers-only-in-headers responses (grpc-status as an HTTP header, no trailer + # frame) remain valid per the grpc-web contract + channel = _channel( + FakeSender(status=200, headers={"grpc-status": "0"}, body=_frame(b"reply-bytes")) + ) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert asyncio.run(mc(b"q")) == b"reply-bytes" + + def test_stream_stream_error_recommends_insert_many_only(): # batch.dynamic()/fixed_size()/rate_limit() do not exist on the async client (the # only one supported under WASM), so the error must not recommend them From 838387b89948704948c558ab7a1e681277fd2d1b Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:32:03 +0200 Subject: [PATCH 12/27] test: pin the grpcio fallback version against the vendored stub gates Fails on drift in either direction: the Emscripten fallback must pass every vendored *_pb2_grpc.py version gate, and the grpc-web shim's FAKE_GRPC_VERSION must equal weaviate.proto.v1._GRPCIO_FALLBACK_VERSION. --- packages/grpc-web/tests/test_shim_install.py | 12 +++++ proto_test/test_proto.py | 46 ++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/packages/grpc-web/tests/test_shim_install.py b/packages/grpc-web/tests/test_shim_install.py index c950da902..0661a46b5 100644 --- a/packages/grpc-web/tests/test_shim_install.py +++ b/packages/grpc-web/tests/test_shim_install.py @@ -109,3 +109,15 @@ async def main(): ) assert result.returncode == 0, result.stderr assert "OK" in result.stdout + + +def test_fake_grpc_version_matches_base_fallback(): + # In-process on purpose: nothing here installs the shim, we only compare the two + # copies of the pinned version. The shim advertises FAKE_GRPC_VERSION as + # grpc.__version__ and the base package falls back to _GRPCIO_FALLBACK_VERSION + # under Emscripten — the vendored stubs' version gates see both, so they must + # never drift apart. + from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION + from weaviate_grpc_web._shim import FAKE_GRPC_VERSION + + assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/proto_test/test_proto.py b/proto_test/test_proto.py index 54dd89a15..723429da4 100644 --- a/proto_test/test_proto.py +++ b/proto_test/test_proto.py @@ -1,4 +1,6 @@ import importlib +import pathlib +import re from importlib.metadata import PackageNotFoundError, version as metadata_version import pytest @@ -98,3 +100,47 @@ def test_get_version_passthrough_when_installed(monkeypatch): monkeypatch.setattr(mod, "metadata_version", lambda pkg: "1.2.3") assert str(mod.get_version("grpcio")) == "1.2.3" assert str(mod.get_version("protobuf")) == "1.2.3" + + +@pytest.mark.skipif( + _INCOMPATIBLE_GRPC_PB, + reason="weaviate.proto.v1 cannot be imported with an incompatible grpcio/protobuf " + "pair (CI version-gate matrix); the gate is covered by test_proto_import and the " + "fallback is exercised in every compatible cell", +) +def test_grpcio_fallback_version_passes_every_vendored_stub_gate(): + """The Emscripten fallback version must satisfy every vendored stub's version gate. + + Under Pyodide ``get_version("grpcio")`` returns ``_GRPCIO_FALLBACK_VERSION`` and the + shim reports it as ``grpc.__version__``, so every vendored ``*_pb2_grpc.py`` whose + import-time gate (``first_version_is_lower``) rejects it would break at import. If + the protos are regenerated with a newer grpcio-tools, this fails until the fallback + (and the grpc-web shim's ``FAKE_GRPC_VERSION``) is bumped to match. + """ + try: + from grpc._utilities import first_version_is_lower + except ImportError: + pytest.skip( + "grpc._utilities.first_version_is_lower is unavailable in this grpcio; " + "newer matrix cells run the comparison" + ) + + fallback = importlib.import_module("weaviate.proto.v1")._GRPCIO_FALLBACK_VERSION + proto_root = pathlib.Path(__file__).resolve().parents[1] / "weaviate" / "proto" / "v1" + stub_files = sorted(proto_root.glob("*/v1/*_pb2_grpc.py")) + assert stub_files, "no vendored *_pb2_grpc.py stubs found" + + gate_pattern = re.compile(r"^GRPC_GENERATED_VERSION = '([^']+)'", re.MULTILINE) + gated = 0 + for stub in stub_files: + match = gate_pattern.search(stub.read_text()) + if match is None: + continue # older codegen (e.g. v4216) emits no version gate + gated += 1 + generated = match.group(1) + assert not first_version_is_lower(fallback, generated), ( + f"{stub.relative_to(proto_root)} requires grpcio>={generated} but " + f"_GRPCIO_FALLBACK_VERSION is {fallback}; bump the fallback (and the " + "grpc-web shim's FAKE_GRPC_VERSION) to match the regenerated stubs" + ) + assert gated > 0, "no stub carried a GRPC_GENERATED_VERSION gate; check the extraction regex" From 412f879395d80e8649c5219c70285f9773e217c6 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:34:57 +0200 Subject: [PATCH 13/27] fix(wasm): reject sync client construction under Emscripten with the async-only error With PyPI httpx the sync REST path previously failed first with an opaque WeaviateStartUpError ConnectError; the shim's clear async-only guidance was only reachable at open_connection_grpc. Raise it at ConnectionSync construction instead. --- test/test_wasm_compat.py | 19 +++++++++++++++++++ weaviate/connect/v4.py | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index dc3355bc9..56ddd17ca 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -10,6 +10,8 @@ import pytest from httpx import ConnectError, ReadTimeout +from weaviate import WeaviateAsyncClient, WeaviateClient +from weaviate.connect.base import ConnectionParams from weaviate.connect.v4 import _ConnectionBase, _exc_detail from weaviate.embedded import _EmbeddedBase from weaviate.exceptions import ( @@ -33,6 +35,23 @@ def test_embedded_platform_check_passes_on_supported_platforms() -> None: _EmbeddedBase.check_supported_platform() # must not raise on this dev platform +def test_sync_client_construction_raises_async_only_under_emscripten(monkeypatch) -> None: + # without the guard the sync client constructs fine and the first REST call fails + # with an opaque ConnectError; the clear async-only error must win, at construction + monkeypatch.setattr(sys, "platform", "emscripten") + with pytest.raises(WeaviateStartUpError, match="async client"): + WeaviateClient(connection_params=ConnectionParams.from_url("http://localhost:8080", 50051)) + + +def test_async_client_construction_allowed_under_emscripten(monkeypatch) -> None: + # the async client is the supported one under WASM — the guard must not catch it + monkeypatch.setattr(sys, "platform", "emscripten") + client = WeaviateAsyncClient( + connection_params=ConnectionParams.from_url("http://localhost:8080", 50051) + ) + assert client is not None + + def _handle_exceptions(e: Exception, error_msg: str = "") -> None: conn = object.__new__(_ConnectionBase) # keep the bare instance's __del__ quiet (it checks these for unclosed connections) diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index 8ff05816b..f80c2385c 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import sys import time from copy import copy from dataclasses import dataclass, field @@ -985,6 +986,23 @@ def resp(res: Response) -> Optional[Dict[str, Any]]: class ConnectionSync(_ConnectionBase): """Connection class used to communicate to a weaviate instance.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + if sys.platform == "emscripten": + # Fail at construction with the async-only message; otherwise the first + # REST call surfaces an opaque ConnectError long before the grpc-web + # shim's own sync guard is reached (wording mirrors the shim's + # _ASYNC_ONLY_MESSAGE). Pre-set the attributes __del__ reads so the + # never-initialized instance is collected quietly. + self._client = None + self._grpc_channel = None + raise WeaviateStartUpError( + "The synchronous client is not supported under WebAssembly/Pyodide. " + "Use an async client (weaviate.use_async_with_local / " + "use_async_with_weaviate_cloud / use_async_with_custom, or " + "WeaviateAsyncClient) instead." + ) + super().__init__(*args, **kwargs) + def connect(self, force: bool = False) -> None: if self._connected and not force: return None From 275680f0b4d37b493ad8d2a045def7ea51b13094 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:56:37 +0200 Subject: [PATCH 14/27] test(pyodide): add an in-Pyodide e2e suite driven by a Node runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run.mjs loads a pinned Pyodide (314.0.4, CPython 3.14) under Node, micropip-installs the two locally built pure wheels (grpcio skipped via its emscripten marker; pydantic_core/cryptography come from the Pyodide distribution — pydantic_core has no wasm wheel on PyPI, which also rules out the 0.28.x line whose bundled pydantic 2.10.6 predates our >=2.12 pin), then awaits e2e.py against Weaviate's core-native /v1/grpc-web endpoint via grpc_path_prefix: connect with live init checks, insert_many, queries/filters/aggregations, multi-tenancy with per-tenant batch insert/delete, error mapping, and the batch.stream()/ experimental() fail-fast. anyio is installed explicitly: Pyodide's httpx recipe drops it but authlib imports it directly. --- .gitignore | 2 + ci/pyodide-e2e/e2e.py | 153 ++++++++++++++++++++++++++++++++++++ ci/pyodide-e2e/package.json | 8 ++ ci/pyodide-e2e/run.mjs | 73 +++++++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 ci/pyodide-e2e/e2e.py create mode 100644 ci/pyodide-e2e/package.json create mode 100644 ci/pyodide-e2e/run.mjs diff --git a/.gitignore b/.gitignore index bc3a48c66..395b51d6c 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ scratch/ *-test.sh *.hdf5 *.jsonl +ci/pyodide-e2e/node_modules/ +ci/pyodide-e2e/package-lock.json diff --git a/ci/pyodide-e2e/e2e.py b/ci/pyodide-e2e/e2e.py new file mode 100644 index 000000000..43b99a903 --- /dev/null +++ b/ci/pyodide-e2e/e2e.py @@ -0,0 +1,153 @@ +"""In-Pyodide e2e for the Weaviate client over core-native grpc-web. + +Executed by ``run.mjs`` inside Pyodide under Node: the runner runs this module's code +(imports below install the grpc shim + fetch transport) and then awaits ``main()`` on +Pyodide's event loop. Plain asserts with one ``OK`` line per step so CI logs are +diagnosable; any failure exits nonzero. + +Deliberately not covered: browser/CORS behaviour (this runs under Node, no CORS layer) +and OIDC auth flows (anonymous access only). +""" + +import os +import warnings + +import weaviate_grpc_web # bootstraps the grpc shim + fetch transport under Emscripten + +import grpc +import weaviate +import weaviate.classes as wvc +from weaviate.classes.config import DataType, Property +from weaviate.classes.query import Filter +from weaviate.classes.tenants import Tenant +from weaviate.exceptions import WeaviateBatchStreamError, WeaviateQueryError + +COLL = "PyodideE2E" +MT_COLL = "PyodideE2ETenants" +# Weaviate core serves grpc-web natively on the REST port under this prefix +# (default-on since 1.38.3), so no proxy sits between the client and the server. +GRPC_WEB_PREFIX = "/v1/grpc-web" + + +def ok(step: str) -> None: + print(f"OK {step}", flush=True) + + +async def main() -> None: + assert weaviate_grpc_web.is_installed(), "grpc shim did not install under Emscripten" + assert getattr(grpc, "__weaviate_grpc_web_shim__", False), "sys.modules['grpc'] is not the shim" + + host = os.environ.get("WEAVIATE_HOST", "localhost") + port = int(os.environ.get("WEAVIATE_PORT", "8090")) + client = weaviate.use_async_with_custom( + http_host=host, + http_port=port, + http_secure=False, + grpc_host=host, + grpc_port=port, + grpc_secure=False, + grpc_path_prefix=GRPC_WEB_PREFIX, + ) + # No skip_init_checks: connect() performs the gRPC health check over grpc-web. + await client.connect() + ok("connect (health check over grpc-web)") + + try: + for name in (COLL, MT_COLL): + if await client.collections.exists(name): + await client.collections.delete(name) + + await client.collections.create( + COLL, + vector_config=wvc.config.Configure.Vectors.self_provided(), + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="idx", data_type=DataType.INT), + ], + ) + ok("collections.create") + + coll = client.collections.get(COLL) + ret = await coll.data.insert_many([{"title": f"article {i}", "idx": i} for i in range(50)]) + assert not ret.has_errors and len(ret.uuids) == 50, f"insert_many errors: {ret.errors}" + ok("insert_many (BatchObjects) = 50") + + res = await coll.query.fetch_objects(limit=100) + assert len(res.objects) == 50, f"fetch_objects got {len(res.objects)}" + ok("query.fetch_objects = 50") + + res = await coll.query.bm25("article", limit=5) + assert len(res.objects) == 5, f"bm25 got {len(res.objects)}" + ok("query.bm25 limit=5 = 5") + + res = await coll.query.fetch_objects( + filters=Filter.by_property("idx").less_than(10), limit=100 + ) + assert len(res.objects) == 10, f"filtered fetch_objects got {len(res.objects)}" + ok("query.fetch_objects filtered idx<10 = 10") + + agg = await coll.aggregate.over_all(total_count=True) + assert agg.total_count == 50, f"aggregate total_count {agg.total_count}" + agg = await coll.aggregate.over_all( + return_metrics=[wvc.query.Metrics("idx").integer(minimum=True, maximum=True)] + ) + idx = agg.properties["idx"] + assert idx.minimum == 0 and idx.maximum == 49, agg.properties + ok("aggregate count=50 min=0 max=49") + + await client.collections.create( + MT_COLL, + vector_config=wvc.config.Configure.Vectors.self_provided(), + properties=[Property(name="title", data_type=DataType.TEXT)], + multi_tenancy_config=wvc.config.Configure.multi_tenancy(enabled=True), + ) + mt = client.collections.get(MT_COLL) + await mt.tenants.create([Tenant(name="t1"), Tenant(name="t2")]) + tenants = await mt.tenants.get() + assert set(tenants.keys()) == {"t1", "t2"}, f"TenantsGet: {set(tenants.keys())}" + ok("multi-tenant create + TenantsGet = {t1, t2}") + + t1 = mt.with_tenant("t1") + ret = await t1.data.insert_many([{"title": f"tenant doc {i}"} for i in range(10)]) + assert not ret.has_errors and len(ret.uuids) == 10, f"tenant insert_many: {ret.errors}" + agg = await t1.aggregate.over_all(total_count=True) + assert agg.total_count == 10, f"tenant aggregate {agg.total_count}" + ok("per-tenant insert_many = 10, aggregate = 10") + + dm = await t1.data.delete_many(where=Filter.by_property("title").like("tenant*")) + assert dm.successful == 10, f"delete_many successful={dm.successful}" + agg = await t1.aggregate.over_all(total_count=True) + assert agg.total_count == 0, f"post-delete aggregate {agg.total_count}" + ok("per-tenant delete_many (BatchDelete) = 10 -> aggregate = 0") + + try: + await client.collections.get("DoesNotExistXyz").query.fetch_objects(limit=1) + raise AssertionError("expected WeaviateQueryError for nonexistent collection") + except WeaviateQueryError as e: + assert "DoesNotExistXyz" in str(e) or "not" in str(e).lower(), str(e) + ok("error mapping: nonexistent collection -> WeaviateQueryError") + + try: + async with client.batch.stream() as batch: + await batch.add_object(collection=COLL, properties={"title": "x", "idx": 999}) + raise AssertionError("batch.stream() did not raise under grpc-web") + except WeaviateBatchStreamError as e: + assert "grpc-web" in str(e) and "insert_many" in str(e), str(e) + ok("batch.stream() -> WeaviateBatchStreamError (clear message)") + + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + async with client.batch.experimental() as batch: + await batch.add_object(collection=COLL, properties={"title": "x", "idx": 999}) + raise AssertionError("batch.experimental() did not raise under grpc-web") + except WeaviateBatchStreamError: + ok("batch.experimental() -> WeaviateBatchStreamError") + + for name in (COLL, MT_COLL): + await client.collections.delete(name) + ok("cleanup") + finally: + await client.close() + + print("PYODIDE E2E: ALL STEPS OK", flush=True) diff --git a/ci/pyodide-e2e/package.json b/ci/pyodide-e2e/package.json new file mode 100644 index 000000000..fbf78f303 --- /dev/null +++ b/ci/pyodide-e2e/package.json @@ -0,0 +1,8 @@ +{ + "name": "weaviate-pyodide-e2e", + "private": true, + "description": "Runs the weaviate-client e2e suite inside Pyodide (WASM) under Node", + "dependencies": { + "pyodide": "314.0.4" + } +} diff --git a/ci/pyodide-e2e/run.mjs b/ci/pyodide-e2e/run.mjs new file mode 100644 index 000000000..01f95d968 --- /dev/null +++ b/ci/pyodide-e2e/run.mjs @@ -0,0 +1,73 @@ +// Runs the Weaviate Python client e2e suite (e2e.py) inside Pyodide (WASM) under Node. +// +// Usage: node run.mjs +// must contain exactly the two locally-built pure wheels: +// weaviate_client-*.whl and weaviate_python_grpc_web-*.whl. +// Env: WEAVIATE_HOST (default localhost), WEAVIATE_PORT (default 8090). +// +// The pinned `pyodide` npm package fixes the interpreter (the 314.x line bundles +// CPython 3.14), so there is no Python version matrix here. micropip installs the two +// local wheels; transitive deps resolve from the Pyodide distribution +// (pydantic/pydantic_core/cryptography ship wasm builds there — pydantic_core has no +// wasm wheel on PyPI) or from PyPI as pure wheels (protobuf), and the base client's +// `grpcio; sys_platform != "emscripten"` marker correctly skips grpcio. +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadPyodide } from "pyodide"; + +if (!process.argv[2]) { + console.error("usage: node run.mjs "); + process.exit(2); +} +const wheelsDir = resolve(process.argv[2]); +const here = dirname(fileURLToPath(import.meta.url)); + +const wheels = readdirSync(wheelsDir) + .filter((f) => f.endsWith(".whl")) + .sort(); // installs weaviate_client before weaviate_python_grpc_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_python_grpc_web-"]; +if ( + wheels.length !== 2 || + !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) +) { + console.error( + `expected exactly one weaviate_client-*.whl and one weaviate_python_grpc_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + ); + process.exit(2); +} + +const pyodide = await loadPyodide({ + env: { + WEAVIATE_HOST: process.env.WEAVIATE_HOST ?? "localhost", + WEAVIATE_PORT: process.env.WEAVIATE_PORT ?? "8090", + }, +}); +console.log( + `pyodide ${pyodide.version} / python ${pyodide.runPython("import sys; sys.version.split()[0]")}`, +); + +await pyodide.loadPackage("micropip"); +const micropip = pyodide.pyimport("micropip"); +// Pyodide's bundled httpx recipe drops httpx's anyio dependency (its fetch-based +// transport needs no sockets), but authlib's httpx_client imports anyio directly — +// without this, `import weaviate` fails with ModuleNotFoundError. +await micropip.install("anyio"); + +pyodide.FS.mkdirTree("/wheels"); +pyodide.mountNodeFS("/wheels", wheelsDir); +for (const wheel of wheels) { + console.log(`micropip install ${wheel}`); + await micropip.install(`emfs:/wheels/${wheel}`); +} + +// Define e2e.py's globals (imports run here, installing the grpc shim), then await +// main() on Pyodide's event loop — asyncio.run() cannot be used inside Pyodide. +pyodide.runPython(readFileSync(resolve(here, "e2e.py"), "utf8")); +try { + await pyodide.runPythonAsync("await main()"); +} catch (err) { + console.error(err); + process.exit(1); +} From 114b5b9265e3c7e14c38c5dce766d02fb0660938 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:56:53 +0200 Subject: [PATCH 15/27] ci: run the Pyodide (WASM) e2e suite against core-native grpc-web MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New pyodide-e2e job beside grpc-web-tests: build both pure wheels, start the async-tests Weaviate (WEAVIATE_139) from the existing ci/docker-compose-async.yml, and run ci/pyodide-e2e/run.mjs under Node 22. No Python matrix — the pinned Pyodide bundle fixes the interpreter. No Envoy/browser: core serves grpc-web natively on the REST port under /v1/grpc-web (default-on since 1.38.3). --- .github/workflows/main.yaml | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e2004af1c..fb0149a47 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -125,6 +125,52 @@ jobs: - name: Run grpc-web package tests run: pytest packages/grpc-web/tests + pyodide-e2e: + name: Run Pyodide (WASM) e2e Tests + runs-on: ubuntu-latest + timeout-minutes: 15 + # No Python matrix: the pinned Pyodide bundle fixes the interpreter (see + # ci/pyodide-e2e/package.json for the exact pin). + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + fetch-tags: true + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.11" + cache: 'pip' # caching pip dependencies + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: "22" + - name: Login to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + if: ${{ !github.event.pull_request.head.repo.fork && github.triggering_actor != 'dependabot[bot]' }} + with: + username: ${{secrets.DOCKER_USERNAME}} + password: ${{secrets.DOCKER_PASSWORD}} + - name: Build pure wheels (base client + grpc-web) + run: | + pip install build + python -m build --wheel --outdir dist . + python -m build --wheel --outdir dist packages/grpc-web + - name: start weaviate + run: | + source ./ci/compose.sh + export WEAVIATE_VERSION=$WEAVIATE_139 + docker compose -f ci/docker-compose-async.yml up -d + wait "http://localhost:8090" + - name: Run the e2e suite inside Pyodide under Node + env: + WEAVIATE_HOST: localhost + WEAVIATE_PORT: "8090" + run: | + npm install --prefix ci/pyodide-e2e + node ci/pyodide-e2e/run.mjs dist + - name: stop weaviate + if: always() + run: docker compose -f ci/docker-compose-async.yml down --remove-orphans + proto-test: name: Run importing protos test runs-on: ubuntu-latest From 493af9d7302d7eacd238ffbe0a8df7179c6e0665 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:47:57 +0200 Subject: [PATCH 16/27] feat(wasm): bootstrap the grpc-web shim from a bare 'import weaviate' A platform-guarded soft import at the top of weaviate/__init__.py makes plain 'import weaviate' work under Pyodide (clear ImportError when the companion is missing); the companion wheel now carries 'anyio ; sys_platform == "emscripten"' so micropip resolves it without a manual install. --- ci/pyodide-e2e/run.mjs | 18 ++- packages/grpc-web/README.md | 14 ++- packages/grpc-web/pyproject.toml | 2 + .../src/weaviate_grpc_web/__init__.py | 11 +- packages/grpc-web/tests/test_single_import.py | 103 ++++++++++++++++++ weaviate/__init__.py | 20 +++- 6 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 packages/grpc-web/tests/test_single_import.py diff --git a/ci/pyodide-e2e/run.mjs b/ci/pyodide-e2e/run.mjs index 01f95d968..4e6ae652c 100644 --- a/ci/pyodide-e2e/run.mjs +++ b/ci/pyodide-e2e/run.mjs @@ -50,10 +50,9 @@ console.log( await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); -// Pyodide's bundled httpx recipe drops httpx's anyio dependency (its fetch-based -// transport needs no sockets), but authlib's httpx_client imports anyio directly — -// without this, `import weaviate` fails with ModuleNotFoundError. -await micropip.install("anyio"); +// anyio (needed because Pyodide's httpx recipe drops it, while authlib imports it +// directly) resolves from the grpc-web wheel's `anyio ; sys_platform == "emscripten"` +// marker — no explicit install here, so the marker stays proven. pyodide.FS.mkdirTree("/wheels"); pyodide.mountNodeFS("/wheels", wheelsDir); @@ -62,6 +61,17 @@ for (const wheel of wheels) { await micropip.install(`emfs:/wheels/${wheel}`); } +// Single-import check: the FIRST weaviate-side import in this interpreter is a bare +// `import weaviate` — the base client must bootstrap the companion (and the shim) itself. +pyodide.runPython(` +import sys +assert "weaviate_grpc_web" not in sys.modules +import weaviate +assert getattr(sys.modules.get("grpc"), "__weaviate_grpc_web_shim__", False), \\ + "bare 'import weaviate' did not install the grpc shim" +print("OK bare 'import weaviate' bootstrapped the grpc shim") +`); + // Define e2e.py's globals (imports run here, installing the grpc shim), then await // main() on Pyodide's event loop — asyncio.run() cannot be used inside Pyodide. pyodide.runPython(readFileSync(resolve(here, "e2e.py"), "utf8")); diff --git a/packages/grpc-web/README.md b/packages/grpc-web/README.md index c63e10bd1..7b1b44632 100644 --- a/packages/grpc-web/README.md +++ b/packages/grpc-web/README.md @@ -35,9 +35,12 @@ transport. ## Usage +With this package installed, a plain `import weaviate` is all you need — under +Emscripten the base client imports `weaviate_grpc_web` itself before anything else, +which installs the shim (and raises a clear error if the package is missing): + ```python -import weaviate_grpc_web # installs the grpc shim under Emscripten (no-op elsewhere) -import weaviate +import weaviate # bootstraps weaviate_grpc_web automatically under Emscripten client = weaviate.use_async_with_local(skip_init_checks=True) await client.connect() @@ -45,6 +48,13 @@ collection = client.collections.get("Article") await collection.query.near_text("hello", limit=3) ``` +Importing the companion explicitly first also works and remains the explicit form: + +```python +import weaviate_grpc_web # installs the grpc shim under Emscripten (no-op elsewhere) +import weaviate +``` + ## Supported / unsupported | Feature | Kind | Status | diff --git a/packages/grpc-web/pyproject.toml b/packages/grpc-web/pyproject.toml index 5cadabf35..0028350f6 100644 --- a/packages/grpc-web/pyproject.toml +++ b/packages/grpc-web/pyproject.toml @@ -17,6 +17,8 @@ version = "0.0.1.dev0" # Emscripten by the `sys_platform != "emscripten"` marker in the base package's deps). dependencies = [ "weaviate-client", + # Pyodide's bundled httpx build omits anyio, but authlib imports it directly. + 'anyio ; sys_platform == "emscripten"', ] [project.urls] diff --git a/packages/grpc-web/src/weaviate_grpc_web/__init__.py b/packages/grpc-web/src/weaviate_grpc_web/__init__.py index ba9fa14b3..c9c12ed68 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/__init__.py +++ b/packages/grpc-web/src/weaviate_grpc_web/__init__.py @@ -5,17 +5,18 @@ runtime) so that the subsequent ``import weaviate`` succeeds and its async gRPC data path runs over grpc-web (``fetch``) instead of HTTP/2 sockets. -Usage under Pyodide:: +Usage under Pyodide (with this package installed, a bare ``import weaviate`` suffices — +the base client imports this package itself under Emscripten before anything else):: - import weaviate_grpc_web # installs the grpc shim (no-op off Emscripten) import weaviate client = weaviate.use_async_with_local(skip_init_checks=True) await client.connect() -The shim is installed automatically only under Emscripten, so importing this package on a -normal CPython install never clobbers a real, working ``grpcio``. Async clients only — -the synchronous client is not supported in the browser. +An explicit ``import weaviate_grpc_web`` before ``import weaviate`` also works and +remains the explicit form. The shim is installed automatically only under Emscripten, so +importing this package on a normal CPython install never clobbers a real, working +``grpcio``. Async clients only — the synchronous client is not supported in the browser. """ import os diff --git a/packages/grpc-web/tests/test_single_import.py b/packages/grpc-web/tests/test_single_import.py new file mode 100644 index 000000000..1bd774224 --- /dev/null +++ b/packages/grpc-web/tests/test_single_import.py @@ -0,0 +1,103 @@ +"""Tests for the single-import hook in the base client (``weaviate/__init__.py``). + +The hook fires on ``sys.platform == "emscripten"`` and (via the companion's bootstrap) +replaces ``sys.modules['grpc']`` process-wide, so each scenario runs in a fresh +subprocess with the platform faked before ``import weaviate`` — the same pattern as +test_shim_install.py / test_httpx_fetch.py's install tests. +""" + +import pathlib +import subprocess +import sys +import textwrap + +_SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") +_REPO_ROOT = str(pathlib.Path(__file__).resolve().parents[3]) + +# CPython derives the _sysconfigdata module name from sys.platform on first use, so a +# faked platform breaks any later sysconfig lookup (pydantic imports zoneinfo, which +# calls sysconfig.get_config_var). Prime the cache before faking. +_PRIME_SYSCONFIG = """ +import sysconfig + +sysconfig.get_config_vars() +""" + + +def _run( + body: str, *, prelude: str = "", path_entry: str = _SRC, no_site: bool = False +) -> subprocess.CompletedProcess: + # -I -S: skip site-packages entirely (plain -I still processes the venv's .pth + # files), so nothing pip-installed is importable — only stdlib plus `path_entry`. + interp = [sys.executable, "-I", "-S"] if no_site else [sys.executable] + script = f"import sys\nsys.path.insert(0, {path_entry!r})\n" + prelude + textwrap.dedent(body) + return subprocess.run([*interp, "-c", script], capture_output=True, text=True) + + +def test_bare_import_weaviate_installs_shim_under_emscripten(): + result = _run( + prelude=_PRIME_SYSCONFIG, + body=""" + import importlib.machinery, types + + sys.platform = "emscripten" + # Pretend httpx is Pyodide's jsfetch build so the companion's bootstrap skips + # the fetch-transport install (there is no pyodide module on CPython). + fake = types.ModuleType("httpx._transports.jsfetch") + fake.__spec__ = importlib.machinery.ModuleSpec("httpx._transports.jsfetch", loader=None) + sys.modules["httpx._transports.jsfetch"] = fake + + import weaviate # the ONLY weaviate-side import: must bootstrap the companion + + assert "weaviate_grpc_web" in sys.modules, "hook did not import the companion" + import weaviate_grpc_web + assert weaviate_grpc_web.is_installed() + import grpc + assert getattr(grpc, "__weaviate_grpc_web_shim__", False) is True + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_without_companion_raises_clear_import_error(): + # No site-packages, so neither weaviate_grpc_web nor grpcio is importable; the repo + # root goes on sys.path so the weaviate package itself is still found. + result = _run( + """ + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert "weaviate-python-grpc-web" in str(e), str(e) + assert "WebAssembly/Pyodide" in str(e), str(e) + print("OK") + else: + raise AssertionError("expected ImportError without the companion") + """, + path_entry=_REPO_ROOT, + no_site=True, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout + + +def test_bare_import_with_grpc_present_falls_through_silently(): + # Companion blocked but a real grpc IS importable (grpcio in the dev env): the hook + # must fall through and leave the normal import path untouched. + result = _run( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + sys.modules["weaviate_grpc_web"] = None # makes its import raise ImportError + + import weaviate + import grpc + + assert not getattr(grpc, "__weaviate_grpc_web_shim__", False) + print("OK") + """, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/weaviate/__init__.py b/weaviate/__init__.py index f3b38dab5..f766c5a4d 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -1,7 +1,25 @@ """Weaviate Python Client Library used to interact with a Weaviate instance.""" -import os import sys + +# Must run before every other import: under Pyodide there is no grpcio wheel, and importing +# the companion installs the pure-Python grpc shim that everything below resolves against. +if sys.platform == "emscripten": + try: + import weaviate_grpc_web # noqa: F401 + except ImportError: + from importlib.util import find_spec + + if find_spec("grpc") is None: + raise ImportError( + "weaviate requires the weaviate-python-grpc-web package under " + "WebAssembly/Pyodide: there is no grpcio wheel for Emscripten, and " + "weaviate-python-grpc-web provides the grpc-web (fetch) transport in its " + "place. Install it (e.g. micropip.install('weaviate-python-grpc-web')) and " + "import weaviate again." + ) from None + +import os from importlib.metadata import PackageNotFoundError, version from typing import Any From 619ee064882d23866ee0be2f0017cf479238a2b9 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:08:42 +0200 Subject: [PATCH 17/27] refactor(web): rename the companion package to weaviate-client-web Three coordinated renames before first publish: distribution weaviate-python-grpc-web -> weaviate-client-web (mirrors the TS @weaviate/web), import module weaviate_grpc_web -> weaviate_client_web, and directory packages/grpc-web -> packages/web. Nothing was ever published under the old name, so no compat aliases are needed; grpc-web stays as the transport-mechanism term. --- .github/workflows/main.yaml | 10 ++++---- ci/pyodide-e2e/e2e.py | 8 ++++--- ci/pyodide-e2e/run.mjs | 14 +++++------ packages/{grpc-web => web}/README.md | 12 +++++----- packages/{grpc-web => web}/pyproject.toml | 4 ++-- .../src/weaviate_client_web}/__init__.py | 2 +- .../src/weaviate_client_web}/_channel.py | 2 +- .../src/weaviate_client_web}/_framing.py | 0 .../src/weaviate_client_web}/_httpx_fetch.py | 0 .../src/weaviate_client_web}/_sender.py | 2 +- .../src/weaviate_client_web}/_shim.py | 4 ++-- .../src/weaviate_client_web}/py.typed | 0 packages/{grpc-web => web}/tests/conftest.py | 0 .../{grpc-web => web}/tests/test_framing.py | 2 +- .../tests/test_httpx_fetch.py | 20 ++++++++-------- .../tests/test_shim_install.py | 24 +++++++++---------- .../tests/test_single_import.py | 16 ++++++------- .../{grpc-web => web}/tests/test_transport.py | 10 ++++---- test/test_batch_async.py | 2 +- test/test_connection_params.py | 6 ++--- weaviate/__init__.py | 8 +++---- weaviate/connect/base.py | 8 +++---- weaviate/connect/helpers.py | 2 +- weaviate/proto/v1/__init__.py | 2 +- 24 files changed, 80 insertions(+), 78 deletions(-) rename packages/{grpc-web => web}/README.md (92%) rename packages/{grpc-web => web}/pyproject.toml (94%) rename packages/{grpc-web/src/weaviate_grpc_web => web/src/weaviate_client_web}/__init__.py (96%) rename packages/{grpc-web/src/weaviate_grpc_web => web/src/weaviate_client_web}/_channel.py (99%) rename packages/{grpc-web/src/weaviate_grpc_web => web/src/weaviate_client_web}/_framing.py (100%) rename packages/{grpc-web/src/weaviate_grpc_web => web/src/weaviate_client_web}/_httpx_fetch.py (100%) rename packages/{grpc-web/src/weaviate_grpc_web => web/src/weaviate_client_web}/_sender.py (96%) rename packages/{grpc-web/src/weaviate_grpc_web => web/src/weaviate_client_web}/_shim.py (98%) rename packages/{grpc-web/src/weaviate_grpc_web => web/src/weaviate_client_web}/py.typed (100%) rename packages/{grpc-web => web}/tests/conftest.py (100%) rename packages/{grpc-web => web}/tests/test_framing.py (97%) rename packages/{grpc-web => web}/tests/test_httpx_fetch.py (96%) rename packages/{grpc-web => web}/tests/test_shim_install.py (85%) rename packages/{grpc-web => web}/tests/test_single_import.py (85%) rename packages/{grpc-web => web}/tests/test_transport.py (97%) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index fb0149a47..925a1c999 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -45,9 +45,9 @@ jobs: cache: 'pip' # caching pip dependencies - run: pip install -r requirements-devel.txt - name: "Ruff lint" - run: ruff check weaviate test mock_tests integration packages/grpc-web + run: ruff check weaviate test mock_tests integration packages/web - name: "Ruff format" - run: ruff format --diff weaviate test mock_tests integration packages/grpc-web + run: ruff format --diff weaviate test mock_tests integration packages/web - name: "Flake 8" run: flake8 weaviate test mock_tests integration - name: "Check release for pypi" @@ -121,9 +121,9 @@ jobs: cache: 'pip' # caching pip dependencies - run: | pip install -r requirements-test.txt -r requirements-devel.txt - pip install -e . -e packages/grpc-web + pip install -e . -e packages/web - name: Run grpc-web package tests - run: pytest packages/grpc-web/tests + run: pytest packages/web/tests pyodide-e2e: name: Run Pyodide (WASM) e2e Tests @@ -153,7 +153,7 @@ jobs: run: | pip install build python -m build --wheel --outdir dist . - python -m build --wheel --outdir dist packages/grpc-web + python -m build --wheel --outdir dist packages/web - name: start weaviate run: | source ./ci/compose.sh diff --git a/ci/pyodide-e2e/e2e.py b/ci/pyodide-e2e/e2e.py index 43b99a903..0ebba1391 100644 --- a/ci/pyodide-e2e/e2e.py +++ b/ci/pyodide-e2e/e2e.py @@ -12,7 +12,7 @@ import os import warnings -import weaviate_grpc_web # bootstraps the grpc shim + fetch transport under Emscripten +import weaviate_client_web # bootstraps the grpc shim + fetch transport under Emscripten import grpc import weaviate @@ -34,8 +34,10 @@ def ok(step: str) -> None: async def main() -> None: - assert weaviate_grpc_web.is_installed(), "grpc shim did not install under Emscripten" - assert getattr(grpc, "__weaviate_grpc_web_shim__", False), "sys.modules['grpc'] is not the shim" + assert weaviate_client_web.is_installed(), "grpc shim did not install under Emscripten" + assert getattr(grpc, "__weaviate_client_web_shim__", False), ( + "sys.modules['grpc'] is not the shim" + ) host = os.environ.get("WEAVIATE_HOST", "localhost") port = int(os.environ.get("WEAVIATE_PORT", "8090")) diff --git a/ci/pyodide-e2e/run.mjs b/ci/pyodide-e2e/run.mjs index 4e6ae652c..3342c9b1a 100644 --- a/ci/pyodide-e2e/run.mjs +++ b/ci/pyodide-e2e/run.mjs @@ -2,7 +2,7 @@ // // Usage: node run.mjs // must contain exactly the two locally-built pure wheels: -// weaviate_client-*.whl and weaviate_python_grpc_web-*.whl. +// weaviate_client-*.whl and weaviate_client_web-*.whl. // Env: WEAVIATE_HOST (default localhost), WEAVIATE_PORT (default 8090). // // The pinned `pyodide` npm package fixes the interpreter (the 314.x line bundles @@ -26,14 +26,14 @@ const here = dirname(fileURLToPath(import.meta.url)); const wheels = readdirSync(wheelsDir) .filter((f) => f.endsWith(".whl")) - .sort(); // installs weaviate_client before weaviate_python_grpc_web, which depends on it -const prefixes = ["weaviate_client-", "weaviate_python_grpc_web-"]; + .sort(); // installs weaviate_client before weaviate_client_web, which depends on it +const prefixes = ["weaviate_client-", "weaviate_client_web-"]; if ( wheels.length !== 2 || !prefixes.every((p) => wheels.some((w) => w.startsWith(p))) ) { console.error( - `expected exactly one weaviate_client-*.whl and one weaviate_python_grpc_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, + `expected exactly one weaviate_client-*.whl and one weaviate_client_web-*.whl in ${wheelsDir}, found: ${JSON.stringify(wheels)}`, ); process.exit(2); } @@ -51,7 +51,7 @@ console.log( await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); // anyio (needed because Pyodide's httpx recipe drops it, while authlib imports it -// directly) resolves from the grpc-web wheel's `anyio ; sys_platform == "emscripten"` +// directly) resolves from the companion wheel's `anyio ; sys_platform == "emscripten"` // marker — no explicit install here, so the marker stays proven. pyodide.FS.mkdirTree("/wheels"); @@ -65,9 +65,9 @@ for (const wheel of wheels) { // `import weaviate` — the base client must bootstrap the companion (and the shim) itself. pyodide.runPython(` import sys -assert "weaviate_grpc_web" not in sys.modules +assert "weaviate_client_web" not in sys.modules import weaviate -assert getattr(sys.modules.get("grpc"), "__weaviate_grpc_web_shim__", False), \\ +assert getattr(sys.modules.get("grpc"), "__weaviate_client_web_shim__", False), \\ "bare 'import weaviate' did not install the grpc shim" print("OK bare 'import weaviate' bootstrapped the grpc shim") `); diff --git a/packages/grpc-web/README.md b/packages/web/README.md similarity index 92% rename from packages/grpc-web/README.md rename to packages/web/README.md index 7b1b44632..800f476ae 100644 --- a/packages/grpc-web/README.md +++ b/packages/web/README.md @@ -1,4 +1,4 @@ -# weaviate-python-grpc-web +# weaviate-client-web A grpc-web / WebAssembly (Pyodide) transport for the [Weaviate Python client](https://github.com/weaviate/weaviate-python-client), so the @@ -36,11 +36,11 @@ transport. ## Usage With this package installed, a plain `import weaviate` is all you need — under -Emscripten the base client imports `weaviate_grpc_web` itself before anything else, +Emscripten the base client imports `weaviate_client_web` itself before anything else, which installs the shim (and raises a clear error if the package is missing): ```python -import weaviate # bootstraps weaviate_grpc_web automatically under Emscripten +import weaviate # bootstraps weaviate_client_web automatically under Emscripten client = weaviate.use_async_with_local(skip_init_checks=True) await client.connect() @@ -51,7 +51,7 @@ await collection.query.near_text("hello", limit=3) Importing the companion explicitly first also works and remains the explicit form: ```python -import weaviate_grpc_web # installs the grpc shim under Emscripten (no-op elsewhere) +import weaviate_client_web # installs the grpc shim under Emscripten (no-op elsewhere) import weaviate ``` @@ -102,7 +102,7 @@ with CORS, or failures become hard to diagnose: ## Testing on CPython -`weaviate_grpc_web.install(force=True)` installs the shim on a normal CPython +`weaviate_client_web.install(force=True)` installs the shim on a normal CPython interpreter (run it in a fresh process, before importing `weaviate`). Inject a sender -with `weaviate_grpc_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the +with `weaviate_client_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the transport against an Envoy/vanguard transcoder without a browser. diff --git a/packages/grpc-web/pyproject.toml b/packages/web/pyproject.toml similarity index 94% rename from packages/grpc-web/pyproject.toml rename to packages/web/pyproject.toml index 0028350f6..95792c315 100644 --- a/packages/grpc-web/pyproject.toml +++ b/packages/web/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=65", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "weaviate-python-grpc-web" +name = "weaviate-client-web" description = "grpc-web / WASM (Pyodide) transport for the Weaviate Python client" readme = "README.md" requires-python = ">=3.10" @@ -29,4 +29,4 @@ Tracker = "https://github.com/weaviate/weaviate-python-client/issues" where = ["src"] [tool.setuptools.package-data] -weaviate_grpc_web = ["py.typed"] +weaviate_client_web = ["py.typed"] diff --git a/packages/grpc-web/src/weaviate_grpc_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py similarity index 96% rename from packages/grpc-web/src/weaviate_grpc_web/__init__.py rename to packages/web/src/weaviate_client_web/__init__.py index c9c12ed68..f0a481d19 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/__init__.py +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -13,7 +13,7 @@ client = weaviate.use_async_with_local(skip_init_checks=True) await client.connect() -An explicit ``import weaviate_grpc_web`` before ``import weaviate`` also works and +An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and remains the explicit form. The shim is installed automatically only under Emscripten, so importing this package on a normal CPython install never clobbers a real, working ``grpcio``. Async clients only — the synchronous client is not supported in the browser. diff --git a/packages/grpc-web/src/weaviate_grpc_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py similarity index 99% rename from packages/grpc-web/src/weaviate_grpc_web/_channel.py rename to packages/web/src/weaviate_client_web/_channel.py index aef41241f..64d4523ed 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -178,7 +178,7 @@ async def _unary( "content-type": "application/grpc-web+proto", "accept": "application/grpc-web+proto", "x-grpc-web": "1", - "x-user-agent": "weaviate-python-grpc-web", + "x-user-agent": "weaviate-client-web", } _fold_metadata(headers, metadata) if timeout is not None: diff --git a/packages/grpc-web/src/weaviate_grpc_web/_framing.py b/packages/web/src/weaviate_client_web/_framing.py similarity index 100% rename from packages/grpc-web/src/weaviate_grpc_web/_framing.py rename to packages/web/src/weaviate_client_web/_framing.py diff --git a/packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py similarity index 100% rename from packages/grpc-web/src/weaviate_grpc_web/_httpx_fetch.py rename to packages/web/src/weaviate_client_web/_httpx_fetch.py diff --git a/packages/grpc-web/src/weaviate_grpc_web/_sender.py b/packages/web/src/weaviate_client_web/_sender.py similarity index 96% rename from packages/grpc-web/src/weaviate_grpc_web/_sender.py rename to packages/web/src/weaviate_client_web/_sender.py index 2a879ec49..d41f9f6c0 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_sender.py +++ b/packages/web/src/weaviate_client_web/_sender.py @@ -2,7 +2,7 @@ A *sender* is ``async def sender(url, headers, body, timeout) -> (status, headers, body)``. The default uses ``pyodide.http.pyfetch`` (browser fetch); a sender can be injected for -testing or for non-browser runtimes via :func:`weaviate_grpc_web.set_sender`. +testing or for non-browser runtimes via :func:`weaviate_client_web.set_sender`. """ from typing import Awaitable, Callable, Dict, Optional, Tuple diff --git a/packages/grpc-web/src/weaviate_grpc_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py similarity index 98% rename from packages/grpc-web/src/weaviate_grpc_web/_shim.py rename to packages/web/src/weaviate_client_web/_shim.py index c8226cecc..dbf0d79c6 100644 --- a/packages/grpc-web/src/weaviate_grpc_web/_shim.py +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -27,7 +27,7 @@ # it makes the stub's import-time version gate pass. See weaviate/proto/v1/__init__.py. FAKE_GRPC_VERSION = "1.72.1" -_SHIM_MARKER = "__weaviate_grpc_web_shim__" +_SHIM_MARKER = "__weaviate_client_web_shim__" class StatusCode(enum.Enum): @@ -159,7 +159,7 @@ def first_version_is_lower(_version: str, _other: str) -> bool: _ASYNC_ONLY_MESSAGE = ( - "weaviate-python-grpc-web provides an asynchronous-only gRPC transport under " + "weaviate-client-web provides an asynchronous-only gRPC transport under " "WebAssembly/Pyodide. Use an async client (weaviate.use_async_with_local / " "use_async_with_weaviate_cloud / use_async_with_custom, or WeaviateAsyncClient); " "the synchronous client is not supported in the browser." diff --git a/packages/grpc-web/src/weaviate_grpc_web/py.typed b/packages/web/src/weaviate_client_web/py.typed similarity index 100% rename from packages/grpc-web/src/weaviate_grpc_web/py.typed rename to packages/web/src/weaviate_client_web/py.typed diff --git a/packages/grpc-web/tests/conftest.py b/packages/web/tests/conftest.py similarity index 100% rename from packages/grpc-web/tests/conftest.py rename to packages/web/tests/conftest.py diff --git a/packages/grpc-web/tests/test_framing.py b/packages/web/tests/test_framing.py similarity index 97% rename from packages/grpc-web/tests/test_framing.py rename to packages/web/tests/test_framing.py index 320aff7f6..1bfa90f2e 100644 --- a/packages/grpc-web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -2,7 +2,7 @@ import pytest -from weaviate_grpc_web._framing import ( +from weaviate_client_web._framing import ( encode_message, iter_frames, parse_trailers, diff --git a/packages/grpc-web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py similarity index 96% rename from packages/grpc-web/tests/test_httpx_fetch.py rename to packages/web/tests/test_httpx_fetch.py index f6c866100..119c4bcf5 100644 --- a/packages/grpc-web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -18,7 +18,7 @@ import httpx import pytest -from weaviate_grpc_web._httpx_fetch import _fetch_handle_async_request +from weaviate_client_web._httpx_fetch import _fetch_handle_async_request _SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") @@ -289,7 +289,7 @@ def test_crlf_in_header_value_rejected(fake_pyfetch): def test_platform_jsfetch_detection(monkeypatch): import importlib.machinery - from weaviate_grpc_web._httpx_fetch import _platform_httpx_has_fetch_support + from weaviate_client_web._httpx_fetch import _platform_httpx_has_fetch_support # the dev environment runs PyPI httpx (httpcore-based): no jsfetch transport assert _platform_httpx_has_fetch_support() is False @@ -338,7 +338,7 @@ def test_force_install_routes_async_client_through_pyfetch(): prelude=_FAKE_PYODIDE_PRELUDE, body=""" import asyncio, httpx - from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed install_fetch_transport(force=True) assert is_fetch_transport_installed() @@ -365,7 +365,7 @@ def test_install_without_force_is_noop_off_emscripten(): assert sys.platform != "emscripten" import httpx before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed install_fetch_transport() assert not is_fetch_transport_installed() assert httpx.AsyncHTTPTransport.handle_async_request is before @@ -381,7 +381,7 @@ def test_force_install_is_idempotent(): prelude=_FAKE_PYODIDE_PRELUDE, body=""" import httpx - from weaviate_grpc_web import install_fetch_transport + from weaviate_client_web import install_fetch_transport install_fetch_transport(force=True) patched = httpx.AsyncHTTPTransport.handle_async_request install_fetch_transport(force=True) @@ -399,7 +399,7 @@ def test_sync_transport_left_untouched(): body=""" import httpx sync_before = httpx.HTTPTransport.handle_request - from weaviate_grpc_web import install_fetch_transport + from weaviate_client_web import install_fetch_transport install_fetch_transport(force=True) assert httpx.HTTPTransport.handle_request is sync_before print("OK") @@ -415,7 +415,7 @@ def test_uninstall_restores_original_transport(): body=""" import httpx before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_grpc_web import ( + from weaviate_client_web import ( install_fetch_transport, is_fetch_transport_installed, uninstall_fetch_transport, @@ -439,7 +439,7 @@ def test_patched_method_carries_sentinel(): prelude=_FAKE_PYODIDE_PRELUDE, body=""" import httpx - from weaviate_grpc_web import install_fetch_transport + from weaviate_client_web import install_fetch_transport assert not getattr( httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False ) @@ -461,7 +461,7 @@ def test_force_install_without_pyodide_fails_fast(): """ import httpx before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed try: install_fetch_transport(force=True) except ModuleNotFoundError: @@ -492,7 +492,7 @@ def test_emscripten_with_platform_jsfetch_skips_install(): import httpx before = httpx.AsyncHTTPTransport.handle_async_request - from weaviate_grpc_web import install_fetch_transport, is_fetch_transport_installed + from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed install_fetch_transport() # no force: platform transport must win assert not is_fetch_transport_installed() assert httpx.AsyncHTTPTransport.handle_async_request is before diff --git a/packages/grpc-web/tests/test_shim_install.py b/packages/web/tests/test_shim_install.py similarity index 85% rename from packages/grpc-web/tests/test_shim_install.py rename to packages/web/tests/test_shim_install.py index 0661a46b5..cdcbed56d 100644 --- a/packages/grpc-web/tests/test_shim_install.py +++ b/packages/web/tests/test_shim_install.py @@ -20,19 +20,19 @@ def _run(body: str) -> subprocess.CompletedProcess: def test_import_weaviate_under_shim(): result = _run( """ - import weaviate_grpc_web - assert weaviate_grpc_web.install(force=True) is True - assert weaviate_grpc_web.is_installed() + import weaviate_client_web + assert weaviate_client_web.install(force=True) is True + assert weaviate_client_web.is_installed() import grpc - assert getattr(grpc, "__weaviate_grpc_web_shim__", False) is True + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True assert grpc.__version__ == "1.72.1" assert grpc._utilities.first_version_is_lower("1.0.0", "2.0.0") is False from grpc.aio._typing import ChannelArgumentType # noqa: F401 import weaviate # must not raise even though grpcio is shimmed from weaviate.proto.v1 import weaviate_pb2_grpc - from weaviate_grpc_web import GrpcWebChannel + from weaviate_client_web import GrpcWebChannel ch = GrpcWebChannel("localhost:50051", secure=False) stub = weaviate_pb2_grpc.WeaviateStub(ch) @@ -50,8 +50,8 @@ def test_import_weaviate_under_shim(): def test_sync_channel_factory_raises_async_only(): result = _run( """ - import weaviate_grpc_web - weaviate_grpc_web.install(force=True) + import weaviate_client_web + weaviate_client_web.install(force=True) import grpc try: grpc.insecure_channel("localhost:50051") @@ -71,8 +71,8 @@ def test_real_proto_unary_round_trip_under_shim(): """ import asyncio import struct - import weaviate_grpc_web - weaviate_grpc_web.install(force=True) + import weaviate_client_web + weaviate_client_web.install(force=True) import weaviate # noqa: F401 from weaviate.proto.v1 import tenants_pb2, weaviate_pb2_grpc @@ -90,8 +90,8 @@ async def sender(url, headers, body_in, timeout): assert url.endswith("/weaviate.v1.Weaviate/TenantsGet") return 200, {}, body - weaviate_grpc_web.set_sender(sender) - from weaviate_grpc_web import GrpcWebChannel + weaviate_client_web.set_sender(sender) + from weaviate_client_web import GrpcWebChannel ch = GrpcWebChannel("localhost:50051", secure=False) stub = weaviate_pb2_grpc.WeaviateStub(ch) @@ -118,6 +118,6 @@ def test_fake_grpc_version_matches_base_fallback(): # under Emscripten — the vendored stubs' version gates see both, so they must # never drift apart. from weaviate.proto.v1 import _GRPCIO_FALLBACK_VERSION - from weaviate_grpc_web._shim import FAKE_GRPC_VERSION + from weaviate_client_web._shim import FAKE_GRPC_VERSION assert FAKE_GRPC_VERSION == _GRPCIO_FALLBACK_VERSION diff --git a/packages/grpc-web/tests/test_single_import.py b/packages/web/tests/test_single_import.py similarity index 85% rename from packages/grpc-web/tests/test_single_import.py rename to packages/web/tests/test_single_import.py index 1bd774224..aff239c0d 100644 --- a/packages/grpc-web/tests/test_single_import.py +++ b/packages/web/tests/test_single_import.py @@ -49,11 +49,11 @@ def test_bare_import_weaviate_installs_shim_under_emscripten(): import weaviate # the ONLY weaviate-side import: must bootstrap the companion - assert "weaviate_grpc_web" in sys.modules, "hook did not import the companion" - import weaviate_grpc_web - assert weaviate_grpc_web.is_installed() + assert "weaviate_client_web" in sys.modules, "hook did not import the companion" + import weaviate_client_web + assert weaviate_client_web.is_installed() import grpc - assert getattr(grpc, "__weaviate_grpc_web_shim__", False) is True + assert getattr(grpc, "__weaviate_client_web_shim__", False) is True print("OK") """, ) @@ -62,7 +62,7 @@ def test_bare_import_weaviate_installs_shim_under_emscripten(): def test_bare_import_without_companion_raises_clear_import_error(): - # No site-packages, so neither weaviate_grpc_web nor grpcio is importable; the repo + # No site-packages, so neither weaviate_client_web nor grpcio is importable; the repo # root goes on sys.path so the weaviate package itself is still found. result = _run( """ @@ -70,7 +70,7 @@ def test_bare_import_without_companion_raises_clear_import_error(): try: import weaviate except ImportError as e: - assert "weaviate-python-grpc-web" in str(e), str(e) + assert "weaviate-client-web" in str(e), str(e) assert "WebAssembly/Pyodide" in str(e), str(e) print("OK") else: @@ -90,12 +90,12 @@ def test_bare_import_with_grpc_present_falls_through_silently(): prelude=_PRIME_SYSCONFIG, body=""" sys.platform = "emscripten" - sys.modules["weaviate_grpc_web"] = None # makes its import raise ImportError + sys.modules["weaviate_client_web"] = None # makes its import raise ImportError import weaviate import grpc - assert not getattr(grpc, "__weaviate_grpc_web_shim__", False) + assert not getattr(grpc, "__weaviate_client_web_shim__", False) print("OK") """, ) diff --git a/packages/grpc-web/tests/test_transport.py b/packages/web/tests/test_transport.py similarity index 97% rename from packages/grpc-web/tests/test_transport.py rename to packages/web/tests/test_transport.py index 90268c02a..8767613c2 100644 --- a/packages/grpc-web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -1,7 +1,7 @@ """In-process tests for the grpc-web channel/multicallable. These exercise the transport classes directly (they import their grpc base classes from -``weaviate_grpc_web._shim``, not from ``sys.modules['grpc']``), so no shim install is +``weaviate_client_web._shim``, not from ``sys.modules['grpc']``), so no shim install is needed and the real ``grpc`` in the dev environment is left untouched. """ @@ -11,8 +11,8 @@ import pytest -from weaviate_grpc_web._channel import GrpcWebChannel, set_sender -from weaviate_grpc_web._shim import AioChannel, AioRpcError, StatusCode +from weaviate_client_web._channel import GrpcWebChannel, set_sender +from weaviate_client_web._shim import AioChannel, AioRpcError, StatusCode def _frame(payload: bytes, flag: int = 0x00) -> bytes: @@ -290,7 +290,7 @@ def test_path_prefix_normalized_in_url(raw, expected_url): def test_shim_factory_extracts_path_prefix_option(): - from weaviate_grpc_web._shim import _aio_insecure_channel + from weaviate_client_web._shim import _aio_insecure_channel with_prefix = _aio_insecure_channel( target="h:1", @@ -313,6 +313,6 @@ def test_set_sender_overrides_default(): assert asyncio.run(mc(b"q")) == b"y" finally: # restore the real default so other tests/processes are unaffected - from weaviate_grpc_web._sender import pyfetch_sender + from weaviate_client_web._sender import pyfetch_sender set_sender(pyfetch_sender) diff --git a/test/test_batch_async.py b/test/test_batch_async.py index 4c0d59726..0328106f5 100644 --- a/test/test_batch_async.py +++ b/test/test_batch_async.py @@ -25,7 +25,7 @@ def _bare_batch(**mangled) -> _BatchBaseAsync: def test_start_fails_fast_when_grpc_web_shim_active(monkeypatch) -> None: # over grpc-web the BatchStream RPC would die inside the background tasks (silent # drop / endless flush); _start must raise before any task is created - monkeypatch.setattr(grpc, "__weaviate_grpc_web_shim__", True, raising=False) + monkeypatch.setattr(grpc, "__weaviate_client_web_shim__", True, raising=False) batch = _bare_batch() # the guard runs before any attribute access with pytest.raises(WeaviateBatchStreamError, match="insert_many"): asyncio.run(batch._start()) diff --git a/test/test_connection_params.py b/test/test_connection_params.py index 041079cfc..89b9a4cf4 100644 --- a/test/test_connection_params.py +++ b/test/test_connection_params.py @@ -90,7 +90,7 @@ def fake_insecure_channel(target, options=None, **kwargs): monkeypatch.setattr(base_mod.grpc.aio, "insecure_channel", fake_insecure_channel) # grpc-web mode requires the shim to be active; simulate it being installed. - monkeypatch.setattr(base_mod.grpc, "__weaviate_grpc_web_shim__", True, raising=False) + monkeypatch.setattr(base_mod.grpc, "__weaviate_client_web_shim__", True, raising=False) params = ConnectionParams.from_params( http_host="localhost", @@ -123,8 +123,8 @@ def _grpc_web_params() -> ConnectionParams: def test_grpc_channel_rejects_prefix_without_shim(monkeypatch) -> None: # No grpc-web shim active -> must fail fast instead of silently building a native # grpcio channel that ignores the prefix. - monkeypatch.delattr(base_mod.grpc, "__weaviate_grpc_web_shim__", raising=False) - with pytest.raises(WeaviateInvalidInputError, match="weaviate-python-grpc-web"): + monkeypatch.delattr(base_mod.grpc, "__weaviate_client_web_shim__", raising=False) + with pytest.raises(WeaviateInvalidInputError, match="weaviate-client-web"): _grpc_web_params()._grpc_channel(proxies={}, grpc_msg_size=None, is_async=True) diff --git a/weaviate/__init__.py b/weaviate/__init__.py index f766c5a4d..49c1dbadf 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -6,16 +6,16 @@ # the companion installs the pure-Python grpc shim that everything below resolves against. if sys.platform == "emscripten": try: - import weaviate_grpc_web # noqa: F401 + import weaviate_client_web # noqa: F401 except ImportError: from importlib.util import find_spec if find_spec("grpc") is None: raise ImportError( - "weaviate requires the weaviate-python-grpc-web package under " + "weaviate requires the weaviate-client-web package under " "WebAssembly/Pyodide: there is no grpcio wheel for Emscripten, and " - "weaviate-python-grpc-web provides the grpc-web (fetch) transport in its " - "place. Install it (e.g. micropip.install('weaviate-python-grpc-web')) and " + "weaviate-client-web provides the grpc-web (fetch) transport in its " + "place. Install it (e.g. micropip.install('weaviate-client-web')) and " "import weaviate again." ) from None diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index f64c026c5..fd5bed8e5 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -22,14 +22,14 @@ def _grpc_web_shim_active() -> bool: - """Whether the 'weaviate-python-grpc-web' shim has replaced the grpc module. + """Whether the 'weaviate-client-web' shim has replaced the grpc module. The shim (used under WASM/Pyodide, where there is no grpcio wheel) routes unary RPCs over grpc-web/fetch and cannot do bidirectional streaming. The marker attribute is the documented contract between the two packages — keep all sniffs going through this helper. """ - return getattr(grpc, "__weaviate_grpc_web_shim__", False) is True + return getattr(grpc, "__weaviate_client_web_shim__", False) is True class ProtocolParams(BaseModel): @@ -173,7 +173,7 @@ def _grpc_channel( options.extend(grpc_config.channel_options) # grpc-web mode (prefix set): only valid for an async client, and only when the - # weaviate-python-grpc-web shim has replaced the grpc module (it consumes the + # weaviate-client-web shim has replaced the grpc module (it consumes the # grpc-web.path_prefix option). Fail fast otherwise — a native grpcio channel # would silently ignore the option and route over native gRPC, a confusing # misconfiguration. Nothing is added for native gRPC, so its channel options stay @@ -187,7 +187,7 @@ def _grpc_channel( if not _grpc_web_shim_active(): raise WeaviateInvalidInputError( "grpc_path_prefix enables grpc-web, which requires the " - "'weaviate-python-grpc-web' package (it installs a grpc shim before " + "'weaviate-client-web' package (it installs a grpc shim before " "'import weaviate'); it is not active in this environment" ) options.append(("grpc-web.path_prefix", prefix)) diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index bb9e3d771..a2d87afdb 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -626,7 +626,7 @@ def use_async_with_custom( grpc_path_prefix: Optional base-path prefix for a grpc-web endpoint served on the same host:port as REST (e.g. "/grpc-web"). When set, gRPC requests are sent over grpc-web to ``://:/...`` and sharing - the REST host:port is allowed. Requires the ``weaviate-python-grpc-web`` + the REST host:port is allowed. Requires the ``weaviate-client-web`` package. Defaults to None (native gRPC). Returns: diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index f20e52e05..1ad304222 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -19,7 +19,7 @@ # Fallback grpcio version used only when grpcio is not installed as a distribution. # This happens under Pyodide/Emscripten, where grpcio has no wheel and is excluded # via the `sys_platform != "emscripten"` marker in setup.cfg; the grpc module itself -# is provided there by a pure-Python shim (see the weaviate-python-grpc-web package). +# is provided there by a pure-Python shim (see the weaviate-client-web package). # On every normal install grpcio's metadata is present and the real version is used, so # this branch is not taken. Restricted to grpcio AND to Emscripten, so that a broken or # partial grpcio install on a normal platform (metadata missing) still surfaces as From df96466d2de0a66dc1332bc8bcf8f78270ec46d9 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:40:50 +0200 Subject: [PATCH 18/27] fix(grpc-web): decide the response on HTTP status, not frame parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _handle_response split the body before checking the HTTP status, so any non-grpc-web error response — Weaviate's 404 JSON, an nginx HTML 502, an SPA index.html — failed frame parsing and surfaced as INTERNAL "malformed grpc-web response". That left the whole _status_from_http table dead in production: 502/503/504 never reached UNAVAILABLE, the only code _Retry retries, so transient upstream failures stopped being retried. Parse defensively and let the status decide. A non-200 with no grpc-status maps through _status_from_http and carries the status, the request URL and a body excerpt; a 404 names both candidate causes. A valid grpc-status still wins when a proxy sends one alongside a non-200. Trailer values decode leniently, so a non-ASCII grpc-message no longer destroys the grpc-status travelling with it. The replaced test asserted the mapping with an empty error body, a shape no real server or proxy emits. --- .../web/src/weaviate_client_web/_channel.py | 91 ++++++++++- .../web/src/weaviate_client_web/_framing.py | 10 +- packages/web/tests/test_framing.py | 23 +++ packages/web/tests/test_transport.py | 147 +++++++++++++++++- 4 files changed, 257 insertions(+), 14 deletions(-) diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index 64d4523ed..c3ba77837 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -16,7 +16,7 @@ import base64 import math import urllib.parse -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional from ._framing import encode_message, split_response from ._sender import Sender, pyfetch_sender @@ -214,7 +214,7 @@ async def _unary( ) from exc try: - return self._handle_response(status, resp_headers, body, deserialize) + return self._handle_response(status, resp_headers, body, deserialize, url) except AioRpcError: raise except Exception as exc: # malformed framing / status / payload @@ -229,8 +229,21 @@ def _handle_response( resp_headers: Dict[str, str], body: bytes, deserialize: Callable[[bytes], Any], + url: str = "", ) -> Any: - messages, trailers = split_response(body) if body else ([], {}) + # Frame-parse defensively, and never let a parse failure decide the outcome: + # every real error response carries a body that is NOT grpc-web framing + # (Weaviate's 404 JSON, an nginx error page), so reading '{' or '<' as a frame + # flag would report "malformed grpc-web response" and throw away the HTTP + # status, the URL and the server's own explanation. + messages: List[bytes] = [] + trailers: Dict[str, str] = {} + frame_error: Optional[BaseException] = None + if body: + try: + messages, trailers = split_response(body) + except Exception as exc: + frame_error = exc raw_status = trailers.get("grpc-status") if raw_status is None: @@ -241,11 +254,10 @@ def _handle_response( message = urllib.parse.unquote(raw_message) if raw_status is None: - if http_status != 200: - raise AioRpcError( - code=_status_from_http(http_status), - details=f"HTTP {http_status} from grpc-web endpoint", - ) + # No grpc-status anywhere AND either a non-200 or a body that is not + # grpc-web framing: a gRPC service did not answer this request at all. + if http_status != 200 or frame_error is not None: + raise _non_grpc_web_error(http_status, url, body, frame_error) if messages: # Every grpc-web unary response must carry a grpc-status (trailer frame # or header); a proxy that drops the trailer must not read as success. @@ -259,6 +271,10 @@ def _handle_response( if code is not StatusCode.OK: raise AioRpcError(code=code, details=message) + if frame_error is not None: + # grpc-status said OK but the body will not parse — report what actually + # came back rather than a bare "no message frame". + raise _non_grpc_web_error(http_status, url, body, frame_error) if not messages: details = "grpc-web response contained no message frame" if raw_status is None: @@ -275,6 +291,65 @@ def _handle_response( return deserialize(messages[0]) +_BODY_EXCERPT_LIMIT = 200 + + +def _body_excerpt(body: bytes, limit: int = _BODY_EXCERPT_LIMIT) -> str: + """Render a short, printable, one-line excerpt of a response body for error details. + + The body here is whatever a server or proxy sent — JSON, HTML, or binary — so decode + leniently and drop non-printables: building an error detail must never itself raise. + """ + if not body: + return "" + text = body[:limit].decode("utf-8", "replace") + text = " ".join("".join(ch if ch.isprintable() else " " for ch in text).split()) + if not text: + return f"<{len(body)} non-printable bytes>" + return text + ("..." if len(body) > limit else "") + + +def _non_grpc_web_error( + http_status: int, + url: str, + body: bytes, + frame_error: Optional[BaseException] = None, +) -> AioRpcError: + """Build the error for a response that is not a grpc-web response at all. + + Details always begin with ``HTTP `` and carry the request URL plus a body + excerpt (``weaviate/connect`` matches on that prefix). The status alone rarely + separates "endpoint missing" from "proxy misconfigured"; the server's own body + text usually does. + """ + what = "not a grpc-web response" + if http_status == 200 and frame_error is not None: + what = f"the body is not grpc-web framing ({frame_error})" + parts = [f"HTTP {http_status} from {url or ''}: {what}."] + + if http_status == 404: + # Two candidate causes, and the channel cannot tell them apart (it does not know + # the server version) — name both rather than guess. + parts.append( + "The grpc-web endpoint does not exist at that path: either this Weaviate " + "server predates 1.38.3, the first release to serve grpc-web natively, or " + "the configured grpc-web path prefix is wrong for the proxy in front of it. " + "Weaviate's native prefix is '/v1/grpc-web'." + ) + elif http_status in (502, 503, 504): + parts.append("Weaviate or the proxy in front of it is unavailable.") + elif http_status == 200: + parts.append( + "Something other than a grpc-web endpoint answered — typically a proxy " + "error page or a single-page-app catch-all route serving index.html. Check " + "the grpc-web path prefix (Weaviate's native prefix is '/v1/grpc-web')." + ) + parts.append(f"Response body: {_body_excerpt(body)}") + + code = StatusCode.INTERNAL if http_status == 200 else _status_from_http(http_status) + return AioRpcError(code=code, details=" ".join(parts)) + + def _status_from_http(http_status: int) -> StatusCode: """Map an HTTP status to a gRPC status when no grpc-status is present. diff --git a/packages/web/src/weaviate_client_web/_framing.py b/packages/web/src/weaviate_client_web/_framing.py index 85b6f6972..4449d265b 100644 --- a/packages/web/src/weaviate_client_web/_framing.py +++ b/packages/web/src/weaviate_client_web/_framing.py @@ -42,13 +42,19 @@ def iter_frames(buf: bytes) -> Iterator[Tuple[int, bytes]]: def parse_trailers(raw: bytes) -> Dict[str, str]: - """Parse a trailer frame payload into a lower-cased header dict.""" + """Parse a trailer frame payload into a lower-cased header dict. + + Names are ASCII by spec, but values are decoded leniently: a proxy that does not + percent-encode ``grpc-message``, or a server error quoting a UTF-8 collection / + tenant / property name, puts raw non-ASCII bytes in the trailer. Failing here would + throw away the ``grpc-status`` that came with it. + """ out: Dict[str, str] = {} for line in raw.split(b"\r\n"): if not line: continue key, _, value = line.partition(b":") - out[key.strip().decode("ascii").lower()] = value.strip().decode("ascii") + out[key.strip().decode("ascii").lower()] = value.strip().decode("utf-8", "replace") return out diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index 1bfa90f2e..5bcefe354 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -47,6 +47,29 @@ def test_parse_trailers_lowercases_keys(): assert parsed == {"grpc-status": "0", "grpc-message": "ok"} +def test_parse_trailers_keeps_status_when_message_is_not_ascii(): + # A proxy that does not percent-encode grpc-message, or a server error quoting a + # UTF-8 collection/tenant name, sends raw non-ASCII bytes. Decoding must not raise: + # the grpc-status travelling with it is the part the client acts on. + parsed = parse_trailers("grpc-status:5\r\ngrpc-message:Café not found\r\n".encode("utf-8")) + assert parsed["grpc-status"] == "5" + assert parsed["grpc-message"] == "Café not found" + + +def test_parse_trailers_keeps_status_when_message_is_invalid_utf8(): + # latin-1 (or any non-UTF-8) bytes must degrade to replacement chars, not an error + parsed = parse_trailers(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n") + assert parsed["grpc-status"] == "9" + assert parsed["grpc-message"].startswith("tenant caf") + + +def test_split_response_survives_non_ascii_trailer(): + body = _frame("grpc-status:7\r\ngrpc-message:accès refusé\r\n".encode("utf-8"), 0x80) + messages, trailers = split_response(body) + assert messages == [] + assert trailers["grpc-status"] == "7" + + def test_truncated_frame_raises(): framed = encode_message(b"hello")[:-2] with pytest.raises(ValueError): diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index 8767613c2..3c7d30dcc 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -115,12 +115,151 @@ def test_trailers_only_status_in_http_headers(): assert excinfo.value.code() is StatusCode.UNAUTHENTICATED -def test_http_error_without_grpc_status_maps_to_code(): - channel = _channel(FakeSender(status=403, headers={}, body=b"")) - mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) +# --- non-grpc-web responses ------------------------------------------------------- +# +# Every real error response carries a body, and none of them is grpc-web framing. The +# bodies below are verbatim shapes seen in the wild; an empty error body (the old +# fixture) is the one shape no server or proxy produces, which is why these tests used +# to pass against a code path that never ran. + +# Weaviate's own 404, verbatim from a 1.39.0 server asked for the wrong prefix. +WEAVIATE_404_JSON = ( + b'{"code":404,"message":"path /grpc-web/grpc.health.v1.Health/Check was not found"}' +) +NGINX_502_HTML = ( + b"\r\n502 Bad Gateway\r\n\r\n" + b"

502 Bad Gateway

\r\n
nginx/1.27.3
\r\n" + b"\r\n\r\n" +) +NGINX_404_HTML = ( + b"\r\n404 Not Found\r\n\r\n" + b"

404 Not Found

\r\n
nginx/1.27.3
\r\n" + b"\r\n\r\n" +) +# A single-page app's catch-all route answers 200 with index.html for unknown paths. +SPA_INDEX_HTML = ( + b'\n\n \n My App\n' + b' \n' + b' \n
\n\n' +) + + +def _details_of(status, body, headers=None, path="/grpc.health.v1.Health/Check"): + """Run one request against a canned HTTP response and return the AioRpcError.""" + channel = _channel(FakeSender(status=status, headers=headers or {}, body=body)) + mc = channel.unary_unary(path, lambda x: x, lambda b: b) with pytest.raises(AioRpcError) as excinfo: asyncio.run(mc(b"q")) - assert excinfo.value.code() is StatusCode.PERMISSION_DENIED + return excinfo.value + + +def test_weaviate_404_json_names_both_candidate_causes(): + # A 404 means EITHER the server predates the native /v1/grpc-web endpoint OR the + # configured path prefix is wrong. The channel cannot tell which, so it must say both. + err = _details_of(404, WEAVIATE_404_JSON, {"content-type": "application/json"}) + details = err.details() + + assert err.code() is StatusCode.UNIMPLEMENTED + assert details.startswith("HTTP 404 ") + assert "/grpc.health.v1.Health/Check" in details # the request path + assert "1.38.3" in details # candidate 1: server too old + assert "path prefix" in details # candidate 2: wrong prefix + assert "/v1/grpc-web" in details # the native prefix, spelled out + assert "was not found" in details # the server's own explanation + assert "malformed grpc-web response" not in details + + +def test_nginx_502_maps_to_unavailable_so_the_client_retries(): + # weaviate/retry.py retries UNAVAILABLE and nothing else; a gateway error arriving + # as INTERNAL is silently un-retried, which is the regression this pins. + err = _details_of(502, NGINX_502_HTML) + assert err.code() is StatusCode.UNAVAILABLE + assert err.details().startswith("HTTP 502 ") + assert "502 Bad Gateway" in err.details() + + +@pytest.mark.parametrize("status", [503, 504]) +def test_gateway_errors_are_unavailable(status): + err = _details_of(status, b"upstream down") + assert err.code() is StatusCode.UNAVAILABLE + + +def test_nginx_404_html_is_reported_as_an_http_404(): + err = _details_of(404, NGINX_404_HTML) + assert err.code() is StatusCode.UNIMPLEMENTED + assert err.details().startswith("HTTP 404 ") + assert "404 Not Found" in err.details() + assert "malformed grpc-web response" not in err.details() + + +def test_spa_fallback_html_200_is_distinguishable_from_a_404(): + # An HTTP 200 serving index.html is the other half of a wrong path prefix: the app's + # catch-all route answers instead of Weaviate. It must not read as malformed framing. + err = _details_of(200, SPA_INDEX_HTML) + details = err.details() + + assert details.startswith("HTTP 200 ") + assert "" in details + assert "single-page-app" in details # names the actual cause + assert "malformed grpc-web response" not in details + # distinguishable from the 404 case, not the same generic message + assert details != _details_of(404, NGINX_404_HTML).details() + + +def test_401_json_body_maps_to_unauthenticated(): + err = _details_of(401, b'{"error":[{"message":"anonymous access not enabled"}]}') + assert err.code() is StatusCode.UNAUTHENTICATED + assert err.details().startswith("HTTP 401 ") + assert "anonymous access not enabled" in err.details() + + +def test_403_error_body_reaches_details(): + # regression: the response body is the most actionable part of the error and must + # survive into details() rather than being parsed as frames and discarded + err = _details_of(403, b'{"code":403,"message":"forbidden: rbac denied"}') + assert err.code() is StatusCode.PERMISSION_DENIED + assert "forbidden: rbac denied" in err.details() + + +def test_error_body_excerpt_is_capped(): + err = _details_of(500, b"E" * 5000) + details = err.details() + assert "EEEE" in details + assert details.endswith("...") + assert len(details) < 600 # the 5000-byte body is excerpted, not pasted in + + +def test_binary_error_body_does_not_break_the_error(): + # a proxy answering with a binary payload must not raise UnicodeDecodeError while + # the error message is being built + err = _details_of(502, b"\xff\xfe\x00\x01\x02") + assert err.code() is StatusCode.UNAVAILABLE + assert err.details().startswith("HTTP 502 ") + + +def test_non_200_with_valid_grpc_web_trailers_still_uses_grpc_status(): + # guard on the fix's shape: the HTTP status must not shadow a real grpc-status that + # a proxy shipped alongside a non-200 + err = _details_of(500, _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80)) + assert err.code() is StatusCode.PERMISSION_DENIED + assert err.details() == "denied" + + +def test_non_ascii_grpc_message_preserves_the_status(): + # a trailer carrying raw UTF-8 (an un-percent-encoded proxy, or an error quoting a + # collection name) must not degrade to INTERNAL and lose grpc-status + body = _frame("grpc-status:5\r\ngrpc-message:collection Café not found\r\n".encode(), 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.NOT_FOUND + assert "Caf" in err.details() + + +def test_invalid_utf8_grpc_message_preserves_the_status(): + # latin-1 bytes are not valid UTF-8; the status must still survive + body = _frame(b"grpc-status:9\r\ngrpc-message:tenant caf\xe9 is COLD\r\n", 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.FAILED_PRECONDITION + assert "tenant caf" in err.details() def test_binary_metadata_base64_encoded(): From a7da8c2e21e603ae397aa418bf533cd61687365b Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:41:13 +0200 Subject: [PATCH 19/27] fix: surface failures that previously vanished MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The async token refresher caught only HTTPError, so an OAuthError from the IdP killed the task with no output at all — not before close, not at close, not at exit — and the token silently expired. Catch broadly, re-raise cancellation, and warn if the task dies anyway. Batch flush() checked only __bg_exception, so a BaseException in a background task (grpc.aio raises CancelledError on a cancelled stream) left the queues undrained and flush() spinning forever. Use the liveness check the sync colour already had. __put() could recurse on the same condition. connect() discarded the gRPC code and details and printed firewall and separate-port advice that cannot apply over grpc-web, where gRPC rides the REST port. Report the real status, and on a 404 name both candidates: a server older than 1.38.3, or a wrong grpc_path_prefix. httpx.ConnectTimeout subclasses neither ConnectError nor ReadTimeout, so it escaped the exception taxonomy raw. Map TimeoutException. Under Emscripten the connect timeout bounds the entire fetch, not just connection setup, capping every REST call at 5s no matter what the caller configured. Let it follow the request timeout there; other platforms are unchanged. --- mock_tests/test_auth.py | 84 +++++++++++++++++ test/test_batch_async.py | 89 +++++++++++++++++- test/test_wasm_compat.py | 136 ++++++++++++++++++++++++++- weaviate/collections/batch/async_.py | 53 +++++++++-- weaviate/connect/v4.py | 69 ++++++++++++-- weaviate/exceptions.py | 57 ++++++++++- weaviate/warnings.py | 13 +++ 7 files changed, 481 insertions(+), 20 deletions(-) diff --git a/mock_tests/test_auth.py b/mock_tests/test_auth.py index fc07c904e..95d64858e 100644 --- a/mock_tests/test_auth.py +++ b/mock_tests/test_auth.py @@ -11,6 +11,7 @@ import weaviate from mock_tests.conftest import CLIENT_ID, MOCK_IP, MOCK_PORT, MOCK_PORT_GRPC +from weaviate.connect.v4 import _ConnectionBase from weaviate.exceptions import MissingScopeException ACCESS_TOKEN = "HELLO!IamAnAccessToken" @@ -122,6 +123,89 @@ def handler(request: Request) -> Response: assert token_requests > first # a fresh token was fetched with the credentials +@pytest.mark.asyncio +async def test_token_refresh_survives_non_http_error_async( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server, recwarn +) -> None: + """A refresh failure that is NOT an httpx.HTTPError must warn and keep the refresher alive. + + An IdP rejecting the refresh (400 invalid_grant) makes authlib raise OAuthError. When + only HTTPError was caught, the refresh task died silently — nothing awaits it, so not + even asyncio's 'exception was never retrieved' fired — and every later request 401ed + with nothing pointing at the token refresh. + """ + weaviate_auth_mock.expect_request("/auth").respond_with_response( + Response( + json.dumps({"error": "invalid_grant", "error_description": "refresh token expired"}), + status=400, + content_type="application/json", + ) + ) + weaviate_auth_mock.expect_request( + "/v1/schema", headers={"Authorization": "Bearer " + ACCESS_TOKEN} + ).respond_with_json({"classes": []}) + + async with weaviate.use_async_with_local( + host=MOCK_IP, + port=MOCK_PORT, + grpc_port=MOCK_PORT_GRPC, + auth_credentials=weaviate.auth.AuthBearerToken( + ACCESS_TOKEN, + refresh_token=REFRESH_TOKEN, + expires_in=1, # force an immediate (and failing) refresh + ), + ) as client: + task = getattr(client._connection, "_ConnectionBase__token_refresh_task") # noqa: B009 + assert task is not None + await asyncio.sleep(2.5) # long enough for at least two failed attempts + assert not task.done() # the refresher survived the failure + await client.collections.list_all() # ... and the client still works + + failed = [w for w in recwarn if str(w.message).startswith("Con001")] + assert len(failed) >= 1 + assert "invalid_grant" in str(failed[0].message) + # the task ended by close()'s cancellation, not by dying on the exception + assert [w for w in recwarn if str(w.message).startswith("Con003")] == [] + + +@pytest.mark.asyncio +async def test_token_refresh_death_outside_loop_body_is_surfaced() -> None: + """A refresher that dies where the loop body cannot catch it must still warn. + + Nothing awaits the task and _cancel_background_token_refresh drops the reference, so + the done-callback is the only thing that can observe such a death. + """ + on_done = getattr(_ConnectionBase, "_ConnectionBase__warn_if_token_refresh_died") # noqa: B009 + + async def dies() -> None: + raise ValueError("boom") + + async def forever() -> None: + await asyncio.sleep(3600) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + task = asyncio.get_running_loop().create_task(dies()) + task.add_done_callback(on_done) + await asyncio.sleep(0) + await asyncio.sleep(0) # let the done-callback run + + stopped = [w for w in caught if str(w.message).startswith("Con003")] + assert len(stopped) == 1 + assert "boom" in str(stopped[0].message) + + # a cancelled refresher (the normal close() path) must stay quiet + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + task = asyncio.get_running_loop().create_task(forever()) + task.add_done_callback(on_done) + task.cancel() + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert [w for w in caught if str(w.message).startswith("Con003")] == [] + + @pytest.mark.parametrize("header_name", ["Authorization", "authorization"]) def test_auth_header_priority( recwarn, weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server, header_name: str diff --git a/test/test_batch_async.py b/test/test_batch_async.py index 0328106f5..5ad611b94 100644 --- a/test/test_batch_async.py +++ b/test/test_batch_async.py @@ -1,8 +1,9 @@ """Unit tests for the async batch-stream failure handling. -These pin three behaviors added for WASM/background-failure robustness without needing -a cluster: the grpc-web fail-fast in _start, flush() raising instead of spinning -forever, and _wait() preserving partial results while still raising. +These pin the behaviors added for WASM/background-failure robustness without needing a +cluster: the grpc-web fail-fast in _start, flush() raising instead of spinning forever +(whether or not __bg_exception was set), and _wait() preserving partial results while +still raising. """ import asyncio @@ -10,11 +11,19 @@ import grpc import pytest -from weaviate.collections.batch.async_ import _BatchBaseAsync +from weaviate.collections.batch.async_ import _BatchBaseAsync, _BgTasks from weaviate.collections.batch.base import _BatchDataWrapper from weaviate.exceptions import WeaviateBatchStreamError +class _NotAnException(BaseException): + """Stands in for what grpc.aio can raise past `except Exception` in the wrappers. + + A custom BaseException, not KeyboardInterrupt/SystemExit: asyncio re-raises those two + out of Task.__step and would tear down the test's event loop. + """ + + def _bare_batch(**mangled) -> _BatchBaseAsync: batch = object.__new__(_BatchBaseAsync) for name, value in mangled.items(): @@ -22,6 +31,24 @@ def _bare_batch(**mangled) -> _BatchBaseAsync: return batch +async def _dead_task(mode: str) -> "asyncio.Task[None]": + """A background task that is already done with __bg_exception left unset.""" + + async def forever() -> None: + await asyncio.sleep(3600) + + async def dies() -> None: + raise _NotAnException("boom") + + task = asyncio.get_running_loop().create_task(forever() if mode == "cancel" else dies()) + if mode == "cancel": + task.cancel() + await asyncio.sleep(0) + await asyncio.sleep(0) # let the task reach its end state + assert task.done() + return task + + def test_start_fails_fast_when_grpc_web_shim_active(monkeypatch) -> None: # over grpc-web the BatchStream RPC would die inside the background tasks (silent # drop / endless flush); _start must raise before any task is created @@ -36,6 +63,7 @@ def test_flush_raises_background_exception_instead_of_hanging() -> None: # asyncio.sleep(0.01) forever batch = _bare_batch( bg_exception=RuntimeError("boom"), + bg_tasks=None, batch_objects=[object()], batch_references=[], ) @@ -47,6 +75,59 @@ async def flush_with_deadline() -> None: asyncio.run(flush_with_deadline()) +@pytest.mark.parametrize("mode", ["cancel", "base_exception"]) +def test_flush_raises_when_a_task_dies_without_setting_bg_exception(mode: str) -> None: + # loop_wrapper/recv_wrapper only catch Exception, so a BaseException — e.g. the + # CancelledError grpc.aio raises on a cancelled streaming call — kills a task with + # __bg_exception unset. flush() must notice the dead task, like its sync twin's + # __check_bg_threads_alive(), instead of spinning forever. + async def run() -> None: + loop_task = asyncio.get_running_loop().create_task(asyncio.sleep(3600)) + recv_task = await _dead_task(mode) + + batch = _bare_batch( + bg_exception=None, # nothing recorded: that is the whole point + bg_tasks=_BgTasks(recv=recv_task, loop=loop_task), + batch_objects=[object()], + batch_references=[], + ) + try: + # the deadline makes a regression fail fast instead of hanging CI + await asyncio.wait_for(batch.flush(), timeout=2) + finally: + loop_task.cancel() + + with pytest.raises(WeaviateBatchStreamError, match="background receive task"): + asyncio.run(run()) + + +def test_wait_raises_when_tasks_die_with_data_still_queued() -> None: + # _wait() returning quietly here would report a partial import as a success + class FakeTimeouts: + insert = 1 + + class FakeConnection: + timeout_config = FakeTimeouts() + + async def run() -> None: + recv_task = await _dead_task("base_exception") + loop_task = await _dead_task("base_exception") + + batch = _bare_batch( + bg_exception=None, + bg_tasks=_BgTasks(recv=recv_task, loop=loop_task), + connection=FakeConnection(), + results_for_wrapper=_BatchDataWrapper(), + results_for_wrapper_backup=_BatchDataWrapper(), + batch_objects=[object()], # still queued => the batch did not complete + batch_references=[], + ) + await batch._wait() + + with pytest.raises(WeaviateBatchStreamError, match="background receive task"): + asyncio.run(run()) + + def test_wait_copies_partial_results_before_raising() -> None: # a user catching the background failure must still see what failed class FakeBgTasks: diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index 56ddd17ca..f568d08e9 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -7,19 +7,25 @@ import sys +import grpc import pytest -from httpx import ConnectError, ReadTimeout +from grpc.aio import AioRpcError, Metadata +from httpx import ConnectError, ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout from weaviate import WeaviateAsyncClient, WeaviateClient +from weaviate.config import ConnectionConfig +from weaviate.config import Timeout as TimeoutConfig from weaviate.connect.base import ConnectionParams from weaviate.connect.v4 import _ConnectionBase, _exc_detail from weaviate.embedded import _EmbeddedBase from weaviate.exceptions import ( WeaviateClosedClientError, WeaviateConnectionError, + WeaviateGRPCUnavailableError, WeaviateStartUpError, WeaviateTimeoutError, ) +from weaviate.util import _ServerVersion def test_embedded_raises_explicit_error_under_emscripten(monkeypatch) -> None: @@ -90,3 +96,131 @@ def test_read_timeout_message_includes_context_and_detail() -> None: def test_exc_detail_formats_empty_and_nonempty_strs() -> None: assert _exc_detail(ValueError("boom")) == "ValueError: boom" assert _exc_detail(ConnectError("")) == "ConnectError('')" + + +@pytest.mark.parametrize( + "error", [ConnectTimeout("Request timed out"), WriteTimeout(""), PoolTimeout("")] +) +def test_httpx_timeouts_map_into_the_weaviate_taxonomy(error: Exception) -> None: + # ConnectTimeout/WriteTimeout/PoolTimeout subclass TimeoutException but neither + # ConnectError nor ReadTimeout, so they used to escape as raw httpx errors. Pyodide + # raises ConnectTimeout when the whole fetch promise exceeds the connect timeout. + with pytest.raises(WeaviateTimeoutError) as excinfo: + _handle_exceptions(error, error_msg="Meta endpoint") + assert "Meta endpoint" in str(excinfo.value) + assert type(error).__name__ in str(excinfo.value) + + +def _connection(prefix=None, *, insert: float = 90, query: float = 30) -> _ConnectionBase: + conn = object.__new__(_ConnectionBase) + conn._client = None + conn._grpc_channel = None + conn._weaviate_version = _ServerVersion.from_string("1.36.0") + conn.timeout_config = TimeoutConfig(insert=insert, query=query) + conn._ConnectionBase__connection_config = ConnectionConfig() # type: ignore[attr-defined] + conn._connection_params = ConnectionParams.from_url( + "http://localhost:8080", + grpc_port=8080 if prefix else 50051, + grpc_path_prefix=prefix, + ) + return conn + + +def _get_timeout(conn: _ConnectionBase, method: str, is_gql_query: bool = False): + return getattr(conn, "_ConnectionBase__get_timeout")(method, is_gql_query) # noqa: B009 + + +def _ping_exception(conn: _ConnectionBase, error: Exception) -> None: + getattr(conn, "_ConnectionBase__handle_ping_exception")(error) # noqa: B009 + + +def test_grpc_web_404_names_the_two_real_causes_and_drops_firewall_advice() -> None: + # over grpc-web there is no separate gRPC port and no firewall: REST just succeeded + # against this very host:port. A 404 means the path was not routed. + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNIMPLEMENTED, + Metadata(), + Metadata(), + details="HTTP 404 for /grpc-web/grpc.health.v1.Health/Check: 404 page not found", + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "firewall" not in msg + assert "port (localhost:8080) are correct" not in msg + assert "UNIMPLEMENTED" in msg # the real code, not swallowed + assert "HTTP 404 for /grpc-web/grpc.health.v1.Health/Check" in msg # ... and details + assert "/grpc-web" in msg # the prefix that was actually used + assert "1.38.3" in msg # candidate 1: server too old ... + assert "v1.36.0" in msg # ... shown against the observed server version + assert "/v1/grpc-web" in msg # candidate 2: wrong prefix + + +def test_grpc_web_non_404_error_still_omits_the_native_port_advice() -> None: + conn = _connection(prefix="/grpc-web") + error = AioRpcError( + grpc.StatusCode.UNAVAILABLE, Metadata(), Metadata(), details="HTTP 502 for /grpc-web/..." + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + assert "firewall" not in msg + assert "UNAVAILABLE" in msg + assert "HTTP 502" in msg + assert "skip_init_checks=True" in msg # the still-useful advice is kept + + +def test_native_grpc_message_keeps_its_advice_and_gains_the_real_status() -> None: + conn = _connection() + error = AioRpcError( + grpc.StatusCode.UNAVAILABLE, Metadata(), Metadata(), details="failed to connect" + ) + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, error) + msg = str(excinfo.value) + + # unchanged guidance for native gRPC ... + assert "The gRPC traffic at the specified port is blocked by a firewall." in msg + assert "Please check that the server address and port (localhost:50051) are correct." in msg + # ... plus the error that was previously discarded + assert "UNAVAILABLE" in msg + assert "failed to connect" in msg + + +def test_non_grpc_ping_error_is_still_reported() -> None: + # not every ping failure is an RpcError; those must not lose the generic advice + conn = _connection() + with pytest.raises(WeaviateGRPCUnavailableError) as excinfo: + _ping_exception(conn, ValueError("boom")) + assert "blocked by a firewall" in str(excinfo.value) + + +def test_rest_timeouts_are_capped_at_five_seconds_on_native_platforms() -> None: + assert sys.platform != "emscripten" + conn = _connection() + insert = _get_timeout(conn, "POST") + assert insert.connect == 5.0 and insert.write == 5.0 # httpx defaults, unchanged + assert insert.read == 90 + + +def test_rest_connect_timeout_follows_the_request_timeout_under_emscripten(monkeypatch) -> None: + # under Pyodide the connect timeout bounds the entire fetch promise, so a fixed 5s + # silently caps every request at ~5s of wall clock + monkeypatch.setattr(sys, "platform", "emscripten") + conn = _connection() + + insert = _get_timeout(conn, "POST") + assert insert.connect == 90 and insert.write == 90 + assert insert.read == 90 + + query = _get_timeout(conn, "GET") + assert query.connect == 30 + assert query.read == 30 + + # a configured timeout below the httpx default must not make connecting stricter + short = _get_timeout(_connection(insert=1, query=1), "POST") + assert short.connect == 5.0 + assert short.read == 1 diff --git a/weaviate/collections/batch/async_.py b/weaviate/collections/batch/async_.py index bd89ad54f..21310ab94 100644 --- a/weaviate/collections/batch/async_.py +++ b/weaviate/collections/batch/async_.py @@ -217,6 +217,13 @@ async def _wait(self) -> None: # surface background-task failures instead of returning partial results # as if the batch had succeeded raise self.__bg_exception + if ( + len(self.__batch_objects) > 0 or len(self.__batch_references) > 0 + ) and not self.__all_tasks_alive(): + # the tasks are gone with data still queued and nothing recorded in + # __bg_exception (a BaseException escaping loop_wrapper/recv_wrapper): the + # batch did NOT complete, so do not return as if it had + raise self.__bg_task_death_cause() async def _shutdown(self) -> None: self.__is_stopped.set() @@ -226,7 +233,13 @@ async def __put(self, req: _BatchStreamRequest | None): await asyncio.wait_for(self.__reqs.put(req), timeout=1) return True except asyncio.TimeoutError: - if self.__bg_exception is not None or self.__shutdown_loop.is_set(): + # __all_tasks_alive: if the receiver is gone the queue will never drain again, + # so retrying forever (and recursing once per second) only defers the hang + if ( + self.__bg_exception is not None + or self.__shutdown_loop.is_set() + or not self.__all_tasks_alive() + ): return False return await self.__put(req) @@ -547,10 +560,13 @@ async def flush(self) -> None: """Flush the batch queue and wait for all requests to be finished.""" # bg thread is sending objs+refs automatically, so simply wait for everything to be done while len(self.__batch_objects) > 0 or len(self.__batch_references) > 0: - if self.__bg_exception is not None: - # the background tasks died; nothing will drain the queues, so waiting - # any longer would hang forever - raise self.__bg_exception + # a dead task is the condition to check, not __bg_exception: loop_wrapper / + # recv_wrapper only catch Exception, so a BaseException (notably the + # asyncio.CancelledError grpc.aio raises on a cancelled streaming call) ends + # the task with __bg_exception unset. Nothing then drains the queues, so + # waiting any longer would hang forever. Mirrors the sync colour's + # __check_bg_threads_alive(). + self.__check_bg_tasks_alive() await asyncio.sleep(0.01) async def _add_object( @@ -648,4 +664,29 @@ def __check_bg_tasks_alive(self) -> None: if self.__all_tasks_alive(): return - raise self.__bg_exception or Exception("Batch tasks died unexpectedly") + raise self.__bg_exception or self.__bg_task_death_cause() + + def __bg_task_death_cause(self) -> Exception: + """Explain a background task that ended without setting __bg_exception. + + loop_wrapper/recv_wrapper only catch Exception, so a BaseException — notably the + asyncio.CancelledError grpc.aio raises when a streaming call is cancelled — ends + the task with nothing recorded. Cancellation is re-wrapped rather than re-raised: + a bare CancelledError escaping a public call would look like the caller itself + was cancelled and would slip past the user's `except Exception`. + """ + if self.__bg_tasks is not None: + for name, task in ( + ("receive", self.__bg_tasks.recv), + ("loop", self.__bg_tasks.loop), + ): + if not task.done(): + continue + if task.cancelled(): + return WeaviateBatchStreamError(f"the background {name} task was cancelled") + exc = task.exception() + if isinstance(exc, Exception): + return exc + if exc is not None: + return WeaviateBatchStreamError(f"the background {name} task died with {exc!r}") + return WeaviateBatchStreamError("the background tasks died unexpectedly") diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index f80c2385c..852da8dd9 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -50,6 +50,7 @@ RequestError, Response, Timeout, + TimeoutException, ) from weaviate import __version__ as client_version @@ -351,15 +352,27 @@ async def execute(): def __handle_ping_response(self, res: health_weaviate_pb2.WeaviateHealthCheckResponse) -> None: if res.status != health_weaviate_pb2.WeaviateHealthCheckResponse.SERVING: raise WeaviateGRPCUnavailableError( - f"v{self.server_version}", self._connection_params._grpc_address + f"v{self.server_version}", + self._connection_params._grpc_address, + grpc_path_prefix=self.__grpc_web_prefix(), ) return None def __handle_ping_exception(self, e: Exception) -> None: + # pass the error along: its code()/details() are the only thing that says what + # actually went wrong, and the generic advice is wrong in grpc-web mode (no + # separate gRPC port, no firewall — REST just succeeded against this endpoint) raise WeaviateGRPCUnavailableError( - f"v{self.server_version}", self._connection_params._grpc_address + f"v{self.server_version}", + self._connection_params._grpc_address, + grpc_path_prefix=self.__grpc_web_prefix(), + error=e, ) from e + def __grpc_web_prefix(self) -> Optional[str]: + """The configured grpc-web base path, or None when this is native gRPC.""" + return self._connection_params._grpc_web_path_prefix or None + @property def grpc_stub(self) -> Optional[weaviate_pb2_grpc.WeaviateStub]: if not self.is_connected(): @@ -562,9 +575,9 @@ def _create_background_token_refresh(self, _auth: Optional[_Auth] = None) -> Non if loop is not None: # The async colour refreshes on its own already-running loop: threads # cannot start under WASM/Pyodide and are unnecessary here anyway. - self.__token_refresh_task = loop.create_task( - self.__periodic_token_refresh_async(expires_in, _auth) - ) + task = loop.create_task(self.__periodic_token_refresh_async(expires_in, _auth)) + task.add_done_callback(self.__warn_if_token_refresh_died) + self.__token_refresh_task = task return # sync colour (or async without a running loop): refresh on a daemon thread, @@ -648,6 +661,20 @@ def _cancel_background_token_refresh(self) -> None: self.__token_refresh_task.cancel() self.__token_refresh_task = None + @staticmethod + def __warn_if_token_refresh_died(task: "asyncio.Task[None]") -> None: + """Surface a refresher death that happened outside the loop body's own handler. + + Nothing ever awaits this task and ``_cancel_background_token_refresh`` drops the + reference, so without this callback not even asyncio's "exception was never + retrieved" message fires — the client just silently stops refreshing. + """ + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + _Warnings.token_refresh_stopped(exc) + async def __periodic_token_refresh_async( self, refresh_time: int, _auth: Optional[_Auth] ) -> None: @@ -674,8 +701,15 @@ async def __periodic_token_refresh_async( new_session = await _Auth.aresult(_auth.get_auth_session()) client.token = await new_session.fetch_token() refresh_time = client.token.get("expires_in", 60) - 30 - except HTTPError as exc: - # retry again after one second, might be an unstable connection + except asyncio.CancelledError: + # close() cancels this task — cancellation must never be swallowed + raise + except Exception as exc: + # retry again after one second, might be an unstable connection. + # Catch broadly: an authlib OAuthError (e.g. invalid_grant), a KeyError on + # a missing token_endpoint or a malformed IdP body would otherwise kill the + # task with nobody watching, and every later request would 401 with nothing + # pointing at the token refresh. refresh_time = 1 _Warnings.token_refresh_failed(exc) @@ -706,6 +740,9 @@ def __get_timeout( They specify the times depending on how they expect Weaviate to behave. For example, a query might take longer than an insert or vice versa but, in either case, the user only cares about how long it takes for a response to be received. + The one exception is Emscripten/Pyodide, where the connect timeout bounds the whole fetch + promise rather than connection setup, so it has to follow the request timeout (see below). + https://www.python-httpx.org/advanced/timeouts/ """ timeout = None @@ -717,8 +754,18 @@ def __get_timeout( timeout = self.timeout_config.query elif method == "POST" and not is_gql_query: timeout = self.timeout_config.insert + + connect: float = 5.0 + if sys.platform == "emscripten" and timeout is not None: + # Under Pyodide the REST transport is httpx's AsyncJavascriptFetchTransport, + # where the connect timeout bounds the WHOLE fetch promise (upload + response) + # rather than connection setup only. A fixed 5s there silently caps every + # request at ~5s of wall clock and ignores the configured insert/query + # timeouts. Never below 5s, and untouched off Emscripten, so native platforms + # keep the httpx default for connect/write. + connect = max(connect, float(timeout)) return Timeout( - timeout=5.0, + timeout=connect, read=timeout, pool=self.__connection_config.session_pool_timeout, ) @@ -734,6 +781,12 @@ def __handle_exceptions(self, e: Exception, error_msg: str) -> None: raise WeaviateConnectionError(self.__error_msg_with_detail(error_msg, e)) from e if isinstance(e, ReadTimeout): raise WeaviateTimeoutError(self.__error_msg_with_detail(error_msg, e)) from e + if isinstance(e, TimeoutException): + # ConnectTimeout/WriteTimeout/PoolTimeout subclass TimeoutException but neither + # ConnectError nor ReadTimeout, so they used to escape as raw httpx errors, + # outside the weaviate exception taxonomy (Pyodide raises ConnectTimeout for a + # whole-request timeout). Checked last: the branches above keep their behavior. + raise WeaviateTimeoutError(self.__error_msg_with_detail(error_msg, e)) from e raise e @staticmethod diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index ce0fe6f7e..2024a1258 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -317,6 +317,24 @@ def __init__(self, data: dict): super().__init__(msg) +def _grpc_status_of( + error: Optional[BaseException], +) -> Tuple[Optional[StatusCode], Optional[str]]: + """Return the (code, details) of a gRPC error, or (None, None) if it carries none.""" + if isinstance(error, (AioRpcError, Call)): + try: + return cast(Optional[StatusCode], error.code()), error.details() + except Exception: # a half-initialized call can raise instead of answering + return None, None + return None, None + + +# first Weaviate release that serves grpc-web on the REST port +GRPC_WEB_MIN_SERVER_VERSION = "1.38.3" +# the base path Weaviate itself serves grpc-web from +GRPC_WEB_SERVER_PATH_PREFIX = "/v1/grpc-web" + + class WeaviateGRPCUnavailableError(WeaviateBaseError): """Is raised when a gRPC-backed query is made with no gRPC connection present.""" @@ -324,7 +342,44 @@ def __init__( self, weaviate_version: str = "", grpc_address: Tuple[str, int] = ("not provided", 0), + grpc_path_prefix: Optional[str] = None, + error: Optional[BaseException] = None, ) -> None: + code, details = _grpc_status_of(error) + observed = "" + if code is not None or details: + code_name = code.name if code is not None else "unknown status" + observed = ( + f"\nThe gRPC call failed with: {code_name}{f' - {details}' if details else ''}\n" + ) + + if grpc_path_prefix: + # grpc-web multiplexes gRPC onto the REST host:port under a base path: there + # is no separate gRPC port to unblock, and the client has already talked to + # this exact endpoint over REST — so no firewall/wrong-port advice here. + address = f"{grpc_address[0]}:{grpc_address[1]}" + if code is StatusCode.UNIMPLEMENTED: + reason = f"""The server did not route the grpc-web path '{grpc_path_prefix}' at {address}. Either: +- the server is too old: grpc-web is served from Weaviate {GRPC_WEB_MIN_SERVER_VERSION} onwards, and this server reports {weaviate_version or "an unknown version"}, or +- `grpc_path_prefix` is wrong: Weaviate serves grpc-web at '{GRPC_WEB_SERVER_PATH_PREFIX}'. +""" + else: + reason = f"""This error could be due to one of several reasons: +- grpc-web is not enabled or is incorrectly configured on the server at {address}. +- your connection is unstable or has a high latency. In this case you can: + - increase init-timeout in `weaviate.use_async_with_custom(additional_config=wvc.init.AdditionalConfig(timeout=wvc.init.Timeout(init=X)))` + - disable startup checks by connecting using `skip_init_checks=True` +""" + msg = f""" +Weaviate {weaviate_version} makes use of a high-speed gRPC API as well as a REST API. +Unfortunately, the gRPC health check against Weaviate could not be completed. + +This client is configured for grpc-web (grpc_path_prefix='{grpc_path_prefix}'), which carries gRPC over the REST endpoint {address}; there is no separate gRPC port. + +{reason}{observed}""" + super().__init__(msg) + return + if grpc_address[0] == "not provided": grpc_msg = "Please check the server address and port." else: @@ -340,7 +395,7 @@ def __init__( - your connection is unstable or has a high latency. In this case you can: - increase init-timeout in `weaviate.connect_to_local(additional_config=wvc.init.AdditionalConfig(timeout=wvc.init.Timeout(init=X)))` - disable startup checks by connecting using `skip_init_checks=True` -""" +{observed}""" super().__init__(msg) diff --git a/weaviate/warnings.py b/weaviate/warnings.py index 1c0a1ae0b..053bac634 100644 --- a/weaviate/warnings.py +++ b/weaviate/warnings.py @@ -79,6 +79,19 @@ def token_refresh_failed(exc: Exception) -> None: stacklevel=1, ) + @staticmethod + def token_refresh_stopped(exc: BaseException) -> None: + warnings.warn( + message=f"""Con003: The periodic token refresh stopped unexpectedly. This client will NOT refresh its + access token again and will become unauthenticated once the current token expires (requests will + then fail with 401). Reconnect the client to restart the refresh. + + Exception: {exc!r} + """, + category=UserWarning, + stacklevel=1, + ) + @staticmethod def weaviate_too_old_vs_latest(server_version: str) -> None: warnings.warn( From 524a27a65a1e8a3d92b00ea79c75135e3dafb31f Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:41:21 +0200 Subject: [PATCH 20/27] ci: gate publishing on the grpc-web and pyodide jobs build-and-publish did not depend on grpc-web-tests or pyodide-e2e, so a release tag could publish with both red. --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 925a1c999..240aaa417 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -418,7 +418,7 @@ jobs: build-and-publish: name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI - needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test] + needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package, proto-test, grpc-web-tests, pyodide-e2e] runs-on: ubuntu-latest timeout-minutes: 20 steps: From a344c8965c7c170ac81c990c332e660a2fb6aed1 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:33:51 +0200 Subject: [PATCH 21/27] fix(web): always run REST over the package fetch transport under Emscripten Pyodide's bundled httpx transport dereferences Response.body unconditionally, which is null for HEAD and 204 responses, so data.exists/update/delete_by_id and tenants.exists crashed with a raw AttributeError; it also does not enforce sub-5s REST timeouts end to end. Install the package transport whenever the platform is Emscripten instead of deferring, build the response as a stream so httpx stamps .elapsed (reference_add_many read it), take the request deadline from the read timeout only, and round the AbortSignal up with an int32 cap. The import hook now raises the install hint only when the companion itself is missing and chains any other ImportError, so a broken companion surfaces its own error. The Pyodide e2e gains a transport self-check plus HEAD/204 and reference steps so CI pins the fix. --- ci/pyodide-e2e/e2e.py | 55 +++++- .../src/weaviate_client_web/_httpx_fetch.py | 76 ++++---- packages/web/tests/test_httpx_fetch.py | 163 ++++++++++++++---- packages/web/tests/test_single_import.py | 52 +++++- weaviate/__init__.py | 8 +- weaviate/proto/v1/__init__.py | 13 +- 6 files changed, 275 insertions(+), 92 deletions(-) diff --git a/ci/pyodide-e2e/e2e.py b/ci/pyodide-e2e/e2e.py index 0ebba1391..ae92e40f5 100644 --- a/ci/pyodide-e2e/e2e.py +++ b/ci/pyodide-e2e/e2e.py @@ -10,14 +10,17 @@ """ import os +import uuid import warnings import weaviate_client_web # bootstraps the grpc shim + fetch transport under Emscripten import grpc +import httpx import weaviate import weaviate.classes as wvc -from weaviate.classes.config import DataType, Property +from weaviate.classes.config import DataType, Property, ReferenceProperty +from weaviate.classes.data import DataReference from weaviate.classes.query import Filter from weaviate.classes.tenants import Tenant from weaviate.exceptions import WeaviateBatchStreamError, WeaviateQueryError @@ -38,6 +41,13 @@ async def main() -> None: assert getattr(grpc, "__weaviate_client_web_shim__", False), ( "sys.modules['grpc'] is not the shim" ) + # REST must run through the package's own fetch transport, not Pyodide's bundled + # httpx transport (which cannot read the null body of HEAD / 204 responses). + assert weaviate_client_web.is_fetch_transport_installed(), "fetch transport not installed" + assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ), "httpx.AsyncHTTPTransport is not the package's fetch transport" + ok("self-check: package fetch transport is the active httpx transport") host = os.environ.get("WEAVIATE_HOST", "localhost") port = int(os.environ.get("WEAVIATE_PORT", "8090")) @@ -66,6 +76,7 @@ async def main() -> None: Property(name="title", data_type=DataType.TEXT), Property(name="idx", data_type=DataType.INT), ], + references=[ReferenceProperty(name="related", target_collection=COLL)], ) ok("collections.create") @@ -97,6 +108,40 @@ async def main() -> None: assert idx.minimum == 0 and idx.maximum == 49, agg.properties ok("aggregate count=50 min=0 max=49") + # REST calls answered without a body (HEAD 204/404, PATCH/DELETE 204) and the + # batch-references path, which reads httpx's response.elapsed. + first, second, last = ret.uuids[0], ret.uuids[1], ret.uuids[49] + assert await coll.data.exists(first) is True + assert await coll.data.exists(uuid.uuid4()) is False + ok("data.exists (HEAD 204 / 404) = True / False") + + await coll.data.update(uuid=first, properties={"title": "article 0 (updated)"}) + obj = await coll.query.fetch_object_by_id(first) + assert obj is not None and obj.properties["title"] == "article 0 (updated)", obj + ok("data.update (PATCH 204) -> fetch_object_by_id sees the update") + + refs = await coll.data.reference_add_many( + [ + DataReference( + from_property="related", from_uuid=ret.uuids[i], to_uuid=ret.uuids[i + 1] + ) + for i in range(5) + ] + ) + assert not refs.has_errors, f"reference_add_many errors: {refs.errors}" + assert refs.elapsed_seconds >= 0, refs + ok("data.reference_add_many (REST /batch/references) = 5") + + await coll.data.reference_delete(from_uuid=first, from_property="related", to=second) + ok("data.reference_delete (DELETE 204)") + + assert await coll.data.delete_by_id(last) is True + assert await coll.data.exists(last) is False + # deleting a missing object answers 204 or 404 depending on the server topology; + # either way it is a body-less response the transport must handle + assert isinstance(await coll.data.delete_by_id(last), bool) + ok("data.delete_by_id (DELETE 204; repeat -> 204/404) = True, then bool") + await client.collections.create( MT_COLL, vector_config=wvc.config.Configure.Vectors.self_provided(), @@ -109,6 +154,10 @@ async def main() -> None: assert set(tenants.keys()) == {"t1", "t2"}, f"TenantsGet: {set(tenants.keys())}" ok("multi-tenant create + TenantsGet = {t1, t2}") + assert await mt.tenants.exists("t1") is True + assert await mt.tenants.exists("t404") is False + ok("tenants.exists (HEAD 200 / 404) = True / False") + t1 = mt.with_tenant("t1") ret = await t1.data.insert_many([{"title": f"tenant doc {i}"} for i in range(10)]) assert not ret.has_errors and len(ret.uuids) == 10, f"tenant insert_many: {ret.errors}" @@ -126,8 +175,8 @@ async def main() -> None: await client.collections.get("DoesNotExistXyz").query.fetch_objects(limit=1) raise AssertionError("expected WeaviateQueryError for nonexistent collection") except WeaviateQueryError as e: - assert "DoesNotExistXyz" in str(e) or "not" in str(e).lower(), str(e) - ok("error mapping: nonexistent collection -> WeaviateQueryError") + assert "DoesNotExistXyz" in str(e), str(e) + ok("error mapping: nonexistent collection -> WeaviateQueryError names the collection") try: async with client.batch.stream() as batch: diff --git a/packages/web/src/weaviate_client_web/_httpx_fetch.py b/packages/web/src/weaviate_client_web/_httpx_fetch.py index f09d6bfbe..e3f0ddf4a 100644 --- a/packages/web/src/weaviate_client_web/_httpx_fetch.py +++ b/packages/web/src/weaviate_client_web/_httpx_fetch.py @@ -11,11 +11,11 @@ philosophy as the grpc shim in ``_shim.py``. Responses are fully buffered, which matches how the base client consumes them (JSON bodies, no streaming). -NOTE: Pyodide >= 0.27 distributes a patched httpx whose ``AsyncHTTPTransport`` already -routes through JS ``fetch`` natively (``httpx/_transports/jsfetch.py``) with streaming -support and a proper connect/read timeout split. When that build is detected, installing -is skipped — overwriting it would replace a better implementation. This transport is the -fallback for environments where httpx resolved from PyPI (httpcore + raw sockets). +It installs under Emscripten even when Pyodide's bundled httpx carries its own JS-fetch +transport (``httpx/_transports/jsfetch.py``): that transport reads ``Response.body`` +unconditionally, which is ``null`` for HEAD requests and 204 responses (``data.exists``, +``data.delete_by_id``, ``tenants.exists``, …), and it does not enforce the per-request +read timeout end-to-end. Known divergences from native httpx (acceptable for the weaviate client's usage): - the browser's fetch follows redirects internally, so httpx never sees a 3xx; @@ -23,7 +23,7 @@ - responses are fully buffered (no streaming). """ -import importlib.util +import math import sys from typing import Callable, Dict, Optional @@ -56,6 +56,10 @@ _TIMEOUT_HINTS = ("timeout", "timed out", "abort") +# JS timers take a signed 32-bit millisecond delay; anything larger overflows and fires +# immediately, so a huge timeout would abort every request at once. +_MAX_ABORT_SIGNAL_MS = 2**31 - 1 + async def _read_request_body(request: httpx.Request) -> bytes: try: @@ -65,18 +69,25 @@ async def _read_request_body(request: httpx.Request) -> bytes: def _pick_timeout(request: httpx.Request) -> Optional[float]: - """Pick the effective deadline from httpx's timeout extension. + """Pick the request deadline from httpx's timeout extension: the ``read`` value only. - httpx populates ``extensions['timeout']`` with connect/read/write/pool values; the - read timeout is what the weaviate client configures per request. ``get(...) or`` - chains would silently skip an explicit 0, so check for None instead. + The base client sets ``read`` per request and passes ``read=None`` for "no deadline" + while still carrying a ``pool`` value; ``connect`` is not separable under fetch and a + pool-acquire timeout has no meaning there, so neither may stand in for ``read``. """ timeouts = request.extensions.get("timeout") or {} - for key in ("read", "connect", "pool"): - value = timeouts.get(key) - if value is not None: - return value - return None + return timeouts.get("read") + + +def _abort_signal_ms(timeout: Optional[float]) -> Optional[int]: + """Milliseconds for ``AbortSignal.timeout``; ``None`` means no client-side deadline. + + Zero, negative and non-finite timeouts all mean "no deadline". Rounded up so a + sub-millisecond timeout never becomes an immediate abort. + """ + if timeout is None or not math.isfinite(timeout) or timeout <= 0: + return None + return min(math.ceil(timeout * 1000), _MAX_ABORT_SIGNAL_MS) def _map_fetch_error( @@ -121,19 +132,21 @@ async def _fetch_handle_async_request( # fetch rejects GET/HEAD requests that carry a body kwargs["body"] = body - timeout = _pick_timeout(request) deadline_set = False - if timeout is not None and timeout > 0: + deadline_ms = _abort_signal_ms(_pick_timeout(request)) + if deadline_ms is not None: try: from js import AbortSignal # type: ignore[import-not-found] - kwargs["signal"] = AbortSignal.timeout(int(timeout * 1000)) + kwargs["signal"] = AbortSignal.timeout(deadline_ms) deadline_set = True except Exception: # pragma: no cover - AbortSignal.timeout availability varies pass try: response = await pyfetch(str(request.url), method=request.method, headers=headers, **kwargs) + # A body-less response (HEAD, 204) reads as b"": fetch resolves a null body to + # an empty ArrayBuffer. data = await response.bytes() except OSError as e: # incl. pyodide.http.AbortError raise _map_fetch_error(e, request, deadline_set) from e @@ -146,10 +159,12 @@ async def _fetch_handle_async_request( } except Exception: # pragma: no cover - header shape varies across Pyodide versions resp_headers = {} + # Hand httpx an unread stream, as its own transports do: the client reads it and + # only then stamps ``response.elapsed``, which the batch-references path relies on. return httpx.Response( status_code=int(response.status), headers=resp_headers, - content=data, + stream=httpx.ByteStream(data), request=request, ) @@ -158,34 +173,17 @@ async def _fetch_handle_async_request( _fetch_handle_async_request.__weaviate_fetch_shim__ = True # type: ignore[attr-defined] -def _platform_httpx_has_fetch_support() -> bool: - """True when the running httpx is Pyodide's distributed build. - - That build replaces the httpcore transport with a native JS-fetch one - (httpx/_transports/jsfetch.py), so the weaviate REST path already works without - this shim — and works better (streaming, connect/read timeout split). - """ - try: - return importlib.util.find_spec("httpx._transports.jsfetch") is not None - except (ImportError, ValueError): # pragma: no cover - exotic import states - return False - - def install_fetch_transport(force: bool = False) -> None: """Patch ``httpx.AsyncHTTPTransport`` to send requests through ``fetch``. Installs only under Emscripten unless ``force=True`` (CPython testing, where a - ``pyodide`` stub must be importable), and is skipped when httpx itself already has - fetch support (Pyodide's distributed build). Idempotent. + ``pyodide`` stub must be importable). Idempotent. """ global _installed, _original_handle_async_request if _installed: return - if not force: - if sys.platform != "emscripten": - return - if _platform_httpx_has_fetch_support(): - return + if not force and sys.platform != "emscripten": + return # Fail fast: the handler imports pyfetch per request, so a missing pyodide module # would otherwise surface as a confusing ModuleNotFoundError on the first request. from pyodide.http import pyfetch # type: ignore[import-not-found] # noqa: F401 diff --git a/packages/web/tests/test_httpx_fetch.py b/packages/web/tests/test_httpx_fetch.py index 119c4bcf5..63480311e 100644 --- a/packages/web/tests/test_httpx_fetch.py +++ b/packages/web/tests/test_httpx_fetch.py @@ -18,19 +18,26 @@ import httpx import pytest -from weaviate_client_web._httpx_fetch import _fetch_handle_async_request +from weaviate_client_web._httpx_fetch import ( + _MAX_ABORT_SIGNAL_MS, + _abort_signal_ms, + _fetch_handle_async_request, +) _SRC = str(pathlib.Path(__file__).resolve().parents[1] / "src") class FakeFetchResponse: - def __init__(self, status: int = 200, headers: Optional[Any] = None, body: bytes = b""): + def __init__( + self, status: int = 200, headers: Optional[Any] = None, body: Optional[bytes] = b"" + ): self.status = status self.headers: Any = headers or {} self._body = body async def bytes(self) -> bytes: # noqa: A003 - mirrors pyodide's FetchResponse API - return self._body + # a null JS body (HEAD, 204) resolves to an empty ArrayBuffer, i.e. b"" + return b"" if self._body is None else self._body class FakePyfetch: @@ -55,10 +62,31 @@ def fake_pyfetch(monkeypatch) -> FakePyfetch: return fetch -def _handle(request: httpx.Request) -> httpx.Response: +async def _handle_async(request: httpx.Request) -> httpx.Response: # self is unused by the handler implementation; a bare transport instance suffices transport = httpx.AsyncHTTPTransport.__new__(httpx.AsyncHTTPTransport) - return asyncio.run(_fetch_handle_async_request(transport, request)) + response = await _fetch_handle_async_request(transport, request) + await response.aread() # httpx.AsyncClient reads non-streamed responses the same way + return response + + +def _handle(request: httpx.Request) -> httpx.Response: + return asyncio.run(_handle_async(request)) + + +class _FetchTransport(httpx.AsyncBaseTransport): + """Route an ``httpx.AsyncClient`` through the handler without patching httpx globally.""" + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + return await _fetch_handle_async_request(self, request) # type: ignore[arg-type] + + +def _via_client(method: str, url: str, **kwargs: Any) -> httpx.Response: + async def main() -> httpx.Response: + async with httpx.AsyncClient(transport=_FetchTransport()) as client: + return await client.request(method, url, **kwargs) + + return asyncio.run(main()) def test_basic_get_round_trip(fake_pyfetch): @@ -82,6 +110,44 @@ def test_response_has_request_attached_for_raise_for_status(fake_pyfetch): response.raise_for_status() +@pytest.mark.parametrize("status", [204, 404]) +def test_head_response_without_body_yields_empty_content(fake_pyfetch, status): + # data.exists() / tenants.exists() are HEAD requests answered 204/404 with a null + # body; the transport must hand httpx an empty response, not fail on the missing body + fake_pyfetch.response = FakeFetchResponse(status=status, body=None) + response = _handle(httpx.Request("HEAD", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == status + assert response.content == b"" + assert "body" not in fake_pyfetch.calls[0] + + +def test_delete_204_without_body_yields_empty_content(fake_pyfetch): + # data.delete_by_id() / reference_delete() are answered 204 with a null body + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = _handle(httpx.Request("DELETE", "http://h:8080/v1/objects/A/uuid")) + assert response.status_code == 204 + assert response.content == b"" + + +def test_body_less_response_through_async_client(fake_pyfetch): + # the full httpx.AsyncClient path (stream wrapping + read) on a body-less response + fake_pyfetch.response = FakeFetchResponse(status=204, body=None) + response = _via_client("HEAD", "http://h:8080/v1/objects/A/uuid") + assert response.status_code == 204 + assert response.content == b"" + + +def test_response_through_async_client_exposes_elapsed_and_content(fake_pyfetch): + # the batch-references path reads ``res.elapsed``, which httpx only sets after it + # has read/closed a stream-backed response; a pre-loaded body never gets one + payload = b'[{"result": {"status": "SUCCESS"}}]' + fake_pyfetch.response = FakeFetchResponse(status=200, body=payload) + response = _via_client("POST", "http://h:8080/v1/batch/references", content=b"[]") + assert response.content == payload + assert response.json() == [{"result": {"status": "SUCCESS"}}] + assert response.elapsed.total_seconds() >= 0 + + def test_fetch_managed_request_headers_stripped(fake_pyfetch): request = httpx.Request( "POST", @@ -184,11 +250,22 @@ def test_read_timeout_maps_to_abort_signal_ms(fake_pyfetch, fake_abort_signal): assert fake_pyfetch.calls[0]["signal"] == "signal-30000" -def test_timeout_falls_back_to_connect_then_pool(fake_pyfetch, fake_abort_signal): +def test_read_none_means_no_deadline_even_with_pool_and_connect_set( + fake_pyfetch, fake_abort_signal +): + # what the base client hands over for a non-finite request timeout: read=None with the + # session pool timeout still set; falling back to pool/connect would abort a long + # insert after 5 s + _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 5})) _handle(_request_with_timeout({"connect": 2.0, "read": None, "write": None, "pool": 9.0})) - _handle(_request_with_timeout({"connect": None, "read": None, "write": None, "pool": 9.0})) - assert fake_abort_signal.timeouts == [2000, 9000] - assert [c["signal"] for c in fake_pyfetch.calls] == ["signal-2000", "signal-9000"] + assert fake_abort_signal.timeouts == [] + assert all("signal" not in c for c in fake_pyfetch.calls) + + +def test_read_timeout_alone_sets_the_deadline(fake_pyfetch, fake_abort_signal): + _handle(_request_with_timeout({"connect": None, "read": 7, "write": None, "pool": 5})) + assert fake_abort_signal.timeouts == [7000] + assert fake_pyfetch.calls[0]["signal"] == "signal-7000" def test_no_timeout_extension_sends_no_signal(fake_pyfetch, fake_abort_signal): @@ -215,6 +292,40 @@ def test_zero_timeout_means_no_deadline(fake_pyfetch, fake_abort_signal): assert "signal" not in fake_pyfetch.calls[0] +@pytest.mark.parametrize( + "timeout,expected_ms", + [ + (None, None), + (0, None), + (-1, None), + (float("inf"), None), + (float("nan"), None), + (0.0001, 1), # rounds up: never an immediate AbortSignal.timeout(0) + (30.0, 30_000), + (1e8, _MAX_ABORT_SIGNAL_MS), + (1e10, _MAX_ABORT_SIGNAL_MS), + ], +) +def test_abort_signal_ms_bounds(timeout, expected_ms): + assert _abort_signal_ms(timeout) == expected_ms + + +def test_infinite_timeout_sends_no_signal(fake_pyfetch, fake_abort_signal): + # an inf read deadline reaching the transport: no signal, not an OverflowError + _handle( + _request_with_timeout({"connect": None, "read": float("inf"), "write": None, "pool": 5}) + ) + assert fake_abort_signal.timeouts == [] + assert "signal" not in fake_pyfetch.calls[0] + + +def test_huge_timeout_is_capped_to_int32_ms(fake_pyfetch, fake_abort_signal): + # setTimeout delays above 2^31-1 ms overflow and fire at once, aborting the request + _handle(_request_with_timeout({"connect": None, "read": 1e10, "write": None, "pool": None})) + assert fake_abort_signal.timeouts == [_MAX_ABORT_SIGNAL_MS] + assert fake_pyfetch.calls[0]["signal"] == f"signal-{_MAX_ABORT_SIGNAL_MS}" + + class RaisingPyfetch: def __init__(self, exc: BaseException): self.exc = exc @@ -286,20 +397,6 @@ def test_crlf_in_header_value_rejected(fake_pyfetch): assert fake_pyfetch.calls == [] -def test_platform_jsfetch_detection(monkeypatch): - import importlib.machinery - - from weaviate_client_web._httpx_fetch import _platform_httpx_has_fetch_support - - # the dev environment runs PyPI httpx (httpcore-based): no jsfetch transport - assert _platform_httpx_has_fetch_support() is False - - fake = types.ModuleType("httpx._transports.jsfetch") - fake.__spec__ = importlib.machinery.ModuleSpec("httpx._transports.jsfetch", loader=None) - monkeypatch.setitem(sys.modules, "httpx._transports.jsfetch", fake) - assert _platform_httpx_has_fetch_support() is True - - # --------------------------------------------------------------------------- # Install semantics: these patch httpx.AsyncHTTPTransport globally, so each # scenario runs in a fresh subprocess (same pattern as test_shim_install.py). @@ -476,11 +573,12 @@ def test_force_install_without_pyodide_fails_fast(): assert "OK" in result.stdout -def test_emscripten_with_platform_jsfetch_skips_install(): - # on Pyodide's distributed httpx (jsfetch transport built in), the shim must NOT - # overwrite the platform implementation +def test_emscripten_installs_even_when_platform_httpx_has_jsfetch(): + # Pyodide's bundled httpx ships a jsfetch transport that crashes on body-less + # responses (HEAD / 204); the shim must take over regardless of the httpx build result = _run( - """ + prelude=_FAKE_PYODIDE_PRELUDE, + body=""" import importlib.machinery, sys, types sys.platform = "emscripten" @@ -493,11 +591,14 @@ def test_emscripten_with_platform_jsfetch_skips_install(): import httpx before = httpx.AsyncHTTPTransport.handle_async_request from weaviate_client_web import install_fetch_transport, is_fetch_transport_installed - install_fetch_transport() # no force: platform transport must win - assert not is_fetch_transport_installed() - assert httpx.AsyncHTTPTransport.handle_async_request is before + install_fetch_transport() # no force: the platform alone must trigger it + assert is_fetch_transport_installed() + assert httpx.AsyncHTTPTransport.handle_async_request is not before + assert getattr( + httpx.AsyncHTTPTransport.handle_async_request, "__weaviate_fetch_shim__", False + ) is True print("OK") - """ + """, ) assert result.returncode == 0, result.stderr assert "OK" in result.stdout diff --git a/packages/web/tests/test_single_import.py b/packages/web/tests/test_single_import.py index aff239c0d..562f31e0a 100644 --- a/packages/web/tests/test_single_import.py +++ b/packages/web/tests/test_single_import.py @@ -23,6 +23,19 @@ sysconfig.get_config_vars() """ +# The companion's bootstrap installs the fetch transport under Emscripten and fails fast +# if pyodide.http cannot be imported, so a faked platform needs a stand-in module. +_FAKE_PYODIDE = """ +import types + +_pyodide = types.ModuleType("pyodide") +_http = types.ModuleType("pyodide.http") +_http.pyfetch = None +_pyodide.http = _http +sys.modules["pyodide"] = _pyodide +sys.modules["pyodide.http"] = _http +""" + def _run( body: str, *, prelude: str = "", path_entry: str = _SRC, no_site: bool = False @@ -36,22 +49,16 @@ def _run( def test_bare_import_weaviate_installs_shim_under_emscripten(): result = _run( - prelude=_PRIME_SYSCONFIG, + prelude=_PRIME_SYSCONFIG + _FAKE_PYODIDE, body=""" - import importlib.machinery, types - sys.platform = "emscripten" - # Pretend httpx is Pyodide's jsfetch build so the companion's bootstrap skips - # the fetch-transport install (there is no pyodide module on CPython). - fake = types.ModuleType("httpx._transports.jsfetch") - fake.__spec__ = importlib.machinery.ModuleSpec("httpx._transports.jsfetch", loader=None) - sys.modules["httpx._transports.jsfetch"] = fake import weaviate # the ONLY weaviate-side import: must bootstrap the companion assert "weaviate_client_web" in sys.modules, "hook did not import the companion" import weaviate_client_web assert weaviate_client_web.is_installed() + assert weaviate_client_web.is_fetch_transport_installed() import grpc assert getattr(grpc, "__weaviate_client_web_shim__", False) is True print("OK") @@ -101,3 +108,32 @@ def test_bare_import_with_grpc_present_falls_through_silently(): ) assert result.returncode == 0, result.stderr assert "OK" in result.stdout + + +def test_bare_import_with_broken_companion_surfaces_its_own_error(tmp_path): + # An INSTALLED companion whose import fails (here: a missing dependency of its own) + # must raise that error, not the "install weaviate-client-web" hint — the hint would + # send the user to reinstall a package that is already there. + fake_pkg = tmp_path / "weaviate_client_web" + fake_pkg.mkdir() + (fake_pkg / "__init__.py").write_text( + "raise ModuleNotFoundError(\"No module named 'anyio'\", name='anyio')\n" + ) + result = _run( + prelude=_PRIME_SYSCONFIG, + body=""" + sys.platform = "emscripten" + try: + import weaviate + except ImportError as e: + assert e.name == "anyio", (e.name, str(e)) + assert "anyio" in str(e), str(e) + assert "weaviate-client-web" not in str(e), str(e) + print("OK") + else: + raise AssertionError("expected the companion's own ImportError to surface") + """, + path_entry=str(tmp_path), + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/weaviate/__init__.py b/weaviate/__init__.py index 49c1dbadf..20ccff232 100644 --- a/weaviate/__init__.py +++ b/weaviate/__init__.py @@ -7,9 +7,13 @@ if sys.platform == "emscripten": try: import weaviate_client_web # noqa: F401 - except ImportError: + except ImportError as exc: from importlib.util import find_spec + # Only an absent companion earns the install hint; a companion that is present + # but fails to import (a broken dependency of its own) must surface that error. + if not (isinstance(exc, ModuleNotFoundError) and exc.name == "weaviate_client_web"): + raise if find_spec("grpc") is None: raise ImportError( "weaviate requires the weaviate-client-web package under " @@ -17,7 +21,7 @@ "weaviate-client-web provides the grpc-web (fetch) transport in its " "place. Install it (e.g. micropip.install('weaviate-client-web')) and " "import weaviate again." - ) from None + ) from exc import os from importlib.metadata import PackageNotFoundError, version diff --git a/weaviate/proto/v1/__init__.py b/weaviate/proto/v1/__init__.py index 1ad304222..62b0910f4 100644 --- a/weaviate/proto/v1/__init__.py +++ b/weaviate/proto/v1/__init__.py @@ -16,15 +16,10 @@ from weaviate.exceptions import WeaviateProtobufIncompatibility -# Fallback grpcio version used only when grpcio is not installed as a distribution. -# This happens under Pyodide/Emscripten, where grpcio has no wheel and is excluded -# via the `sys_platform != "emscripten"` marker in setup.cfg; the grpc module itself -# is provided there by a pure-Python shim (see the weaviate-client-web package). -# On every normal install grpcio's metadata is present and the real version is used, so -# this branch is not taken. Restricted to grpcio AND to Emscripten, so that a broken or -# partial grpcio install on a normal platform (metadata missing) still surfaces as -# PackageNotFoundError instead of silently selecting a fallback stub, and so a genuinely -# missing protobuf (required and pure-Python under Pyodide) is never masked. +# grpcio version assumed under Pyodide/Emscripten, where grpcio has no wheel (excluded by +# the `sys_platform != "emscripten"` marker in setup.cfg) and the grpc module is the +# weaviate-client-web shim. Restricted to grpcio AND Emscripten so a broken grpcio install +# elsewhere still raises PackageNotFoundError, and a missing protobuf is never masked. _GRPCIO_FALLBACK_VERSION = "1.72.1" def get_version(pkg: str) -> version.Version: From e9c81bec39582b52f8de17812197c20d2463e19b Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:34:15 +0200 Subject: [PATCH 22/27] fix(web): harden grpc-web framing, deadline encoding and diagnostics Non-finite deadlines send no grpc-timeout header and no client-side wait; finite ones escalate m -> S -> M to stay within the spec's 8 digits and never use H, which Weaviate's transcoder rejects above 8H (values past ~190 years mean no deadline). Trailer blocks accept LF-only lines and non-ASCII keys, an unknown frame flag or a message after the trailer is a framing error, and a unary response carrying more than one message frame is INTERNAL rather than silently truncated. Metadata with CR/LF/NUL is rejected before any I/O. Diagnostics: an HTTP 405 and a missing grpc_path_prefix under Emscripten both name the fix, and a truncated grpc-web body is reported as truncated instead of as a wrong path. Stale cross-references in comments trimmed. --- .../web/src/weaviate_client_web/_channel.py | 127 ++++++++++--- .../web/src/weaviate_client_web/_framing.py | 51 +++-- packages/web/src/weaviate_client_web/_shim.py | 24 +-- packages/web/tests/test_framing.py | 42 ++++- packages/web/tests/test_transport.py | 174 +++++++++++++++++- 5 files changed, 357 insertions(+), 61 deletions(-) diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index c3ba77837..d6dd4b7cc 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -15,10 +15,11 @@ import asyncio import base64 import math +import sys import urllib.parse from typing import Any, Callable, Dict, List, Optional -from ._framing import encode_message, split_response +from ._framing import TruncatedFrameError, UnknownFrameFlagError, encode_message, split_response from ._sender import Sender, pyfetch_sender from ._shim import AioChannel, AioRpcError, StatusCode, status_from_int @@ -36,14 +37,26 @@ def get_sender() -> Sender: return _default_sender -def _encode_timeout(seconds: float) -> str: - """Encode a timeout as a grpc-timeout header value (````).""" - # Round up so we never advertise a shorter deadline than requested (which would risk - # premature server-side cancellation); grpc-timeout takes a positive integer + unit. - millis = max(1, math.ceil(seconds * 1000)) - if millis < 100_000_000: - return f"{millis}m" - return f"{max(1, math.ceil(seconds))}S" +# grpc-timeout is at most 8 digits plus a unit; anything longer is rejected by the server. +_GRPC_TIMEOUT_MAX = 100_000_000 + + +def _encode_timeout(seconds: Optional[float]) -> Optional[str]: + """Encode a timeout as a grpc-timeout header value; ``None`` means no deadline. + + ``None``, non-finite values and anything beyond 99,999,999 minutes (~190 years) carry + no deadline. Rounds up so we never advertise a shorter deadline than requested (which + would risk premature server-side cancellation), moving to a coarser unit + (m -> S -> M) to stay within 8 digits. Hours are never emitted: grpc-web transcoders + (vanguard) reject any H value above 8H with HTTP 400. + """ + if seconds is None or not math.isfinite(seconds): + return None + for amount, unit in ((seconds * 1000, "m"), (seconds, "S"), (seconds / 60, "M")): + value = max(1, math.ceil(amount)) + if value < _GRPC_TIMEOUT_MAX: + return f"{value}{unit}" + return None def _fold_metadata(headers: Dict[str, str], metadata: Any) -> None: @@ -57,9 +70,13 @@ def _fold_metadata(headers: Dict[str, str], metadata: Any) -> None: name = key.lower() if name.endswith("-bin"): raw = value if isinstance(value, (bytes, bytearray)) else str(value).encode() - headers[name] = base64.b64encode(raw).decode("ascii") + text = base64.b64encode(raw).decode("ascii") else: - headers[name] = value if isinstance(value, str) else str(value) + text = value if isinstance(value, str) else str(value) + # This path bypasses h11/grpcio's header validation, so keep their defence here. + if any(c in name or c in text for c in ("\r", "\n", "\0")): + raise ValueError(f"Illegal character in gRPC metadata {name!r}") + headers[name] = text def _header_lookup(headers: Dict[str, str], name: str) -> Optional[str]: @@ -106,8 +123,8 @@ async def __call__( class _UnsupportedStreamMultiCallable: """Placeholder for ``stream_stream`` (bidirectional streaming). - Calling it raises immediately, before the ``async for`` in ``connect/v4.py:1243`` - begins iterating. + Calling it raises immediately, before the ``async for`` in ``connect/v4.py`` begins + iterating. """ def __init__(self, path: str) -> None: @@ -181,16 +198,20 @@ async def _unary( "x-user-agent": "weaviate-client-web", } _fold_metadata(headers, metadata) - if timeout is not None: - headers["grpc-timeout"] = _encode_timeout(timeout) + grpc_timeout = _encode_timeout(timeout) + if grpc_timeout is None: + timeout = None # None / non-finite: no deadline, server- or client-side + else: + headers["grpc-timeout"] = grpc_timeout url = self._base_url + self._path_prefix + path framed = encode_message(payload) # Send. Enforce a client-side deadline (the grpc-timeout header is server-side # only; pyfetch ignores its timeout arg, so without this a stalled request could - # hang forever). Any transport/parse failure is surfaced as AioRpcError so the - # client only ever sees gRPC error types (never a bare ValueError/TimeoutError). + # hang forever). Any transport/parse failure is surfaced as AioRpcError; the only + # non-gRPC error a caller can see is the ValueError from metadata validation + # above, raised before any I/O (as native grpcio does). try: send = self._sender(url, headers, framed, timeout) if timeout is not None: @@ -208,10 +229,10 @@ async def _unary( # str() of transport errors can be empty (e.g. httpx.ConnectError) — always # include the exception type so failures stay diagnosable detail = f"{type(exc).__name__}: {exc}" if str(exc) else repr(exc) - raise AioRpcError( - code=StatusCode.UNAVAILABLE, - details=f"grpc-web transport error for {path}: {detail}", - ) from exc + details = f"grpc-web transport error for {path}: {detail}" + if not self._path_prefix and sys.platform == "emscripten": + details += " " + _no_path_prefix_hint() + raise AioRpcError(code=StatusCode.UNAVAILABLE, details=details) from exc try: return self._handle_response(status, resp_headers, body, deserialize, url) @@ -231,11 +252,9 @@ def _handle_response( deserialize: Callable[[bytes], Any], url: str = "", ) -> Any: - # Frame-parse defensively, and never let a parse failure decide the outcome: - # every real error response carries a body that is NOT grpc-web framing - # (Weaviate's 404 JSON, an nginx error page), so reading '{' or '<' as a frame - # flag would report "malformed grpc-web response" and throw away the HTTP - # status, the URL and the server's own explanation. + # A frame-parse failure must never decide the outcome by itself: real error + # responses carry non-grpc-web bodies (Weaviate's 404 JSON, an nginx page), and + # the HTTP status, URL and the server's own text must survive into the error. messages: List[bytes] = [] trailers: Dict[str, str] = {} frame_error: Optional[BaseException] = None @@ -257,7 +276,7 @@ def _handle_response( # No grpc-status anywhere AND either a non-200 or a body that is not # grpc-web framing: a gRPC service did not answer this request at all. if http_status != 200 or frame_error is not None: - raise _non_grpc_web_error(http_status, url, body, frame_error) + raise _frame_error_to_rpc(http_status, url, body, frame_error) if messages: # Every grpc-web unary response must carry a grpc-status (trailer frame # or header); a proxy that drops the trailer must not read as success. @@ -274,7 +293,12 @@ def _handle_response( if frame_error is not None: # grpc-status said OK but the body will not parse — report what actually # came back rather than a bare "no message frame". - raise _non_grpc_web_error(http_status, url, body, frame_error) + raise _frame_error_to_rpc(http_status, url, body, frame_error) + if len(messages) > 1: + raise AioRpcError( + code=StatusCode.INTERNAL, + details=f"unary grpc-web response carried {len(messages)} message frames", + ) if not messages: details = "grpc-web response contained no message frame" if raw_status is None: @@ -309,22 +333,53 @@ def _body_excerpt(body: bytes, limit: int = _BODY_EXCERPT_LIMIT) -> str: return text + ("..." if len(body) > limit else "") +def _no_path_prefix_hint() -> str: + # Lazy import: this module is imported while ``weaviate/__init__`` is still + # bootstrapping the shim under Emscripten. + from weaviate.exceptions import GRPC_WEB_MIN_SERVER_VERSION, GRPC_WEB_SERVER_PATH_PREFIX + + return ( + "(no grpc_path_prefix set — for Weaviate >= " + f"{GRPC_WEB_MIN_SERVER_VERSION} use use_async_with_custom(..., " + f"grpc_path_prefix='{GRPC_WEB_SERVER_PATH_PREFIX}') on the REST port, or point " + "grpc_host/grpc_port at a grpc-web transcoder)" + ) + + +def _frame_error_to_rpc( + http_status: int, url: str, body: bytes, frame_error: Optional[BaseException] +) -> AioRpcError: + """Choose the error for a body that did not parse as grpc-web frames.""" + if http_status != 200 or isinstance(frame_error, (UnknownFrameFlagError, TruncatedFrameError)): + return _non_grpc_web_error(http_status, url, body, frame_error) + # Well-formed grpc-web up to the point of failure: a grpc-web endpoint answered but + # broke the protocol (compressed frame, message after trailer, …). + return AioRpcError( + code=StatusCode.INTERNAL, + details=f"malformed grpc-web response from {url or ''}: {frame_error}", + ) + + def _non_grpc_web_error( http_status: int, url: str, body: bytes, frame_error: Optional[BaseException] = None, ) -> AioRpcError: - """Build the error for a response that is not a grpc-web response at all. + """Build the error for a response that is not a usable grpc-web response. Details always begin with ``HTTP `` and carry the request URL plus a body excerpt (``weaviate/connect`` matches on that prefix). The status alone rarely separates "endpoint missing" from "proxy misconfigured"; the server's own body text usually does. """ + truncated = isinstance(frame_error, TruncatedFrameError) what = "not a grpc-web response" if http_status == 200 and frame_error is not None: - what = f"the body is not grpc-web framing ({frame_error})" + if truncated: + what = f"the grpc-web body is truncated ({frame_error})" + else: + what = f"the body is not grpc-web framing ({frame_error})" parts = [f"HTTP {http_status} from {url or ''}: {what}."] if http_status == 404: @@ -336,8 +391,20 @@ def _non_grpc_web_error( "the configured grpc-web path prefix is wrong for the proxy in front of it. " "Weaviate's native prefix is '/v1/grpc-web'." ) + elif http_status == 405: + # A 405 can only come from an existing HTTP route: the prefix points at one. + parts.append( + "An HTTP route answered instead of the grpc-web endpoint (method not " + "allowed): the configured grpc-web path prefix is wrong. Weaviate's native " + "prefix is '/v1/grpc-web'." + ) elif http_status in (502, 503, 504): parts.append("Weaviate or the proxy in front of it is unavailable.") + elif http_status == 200 and truncated: + parts.append( + "The response was cut short — a proxy or browser buffering limit, or the " + "connection dropped mid-response." + ) elif http_status == 200: parts.append( "Something other than a grpc-web endpoint answered — typically a proxy " diff --git a/packages/web/src/weaviate_client_web/_framing.py b/packages/web/src/weaviate_client_web/_framing.py index 4449d265b..7e6a3dfc8 100644 --- a/packages/web/src/weaviate_client_web/_framing.py +++ b/packages/web/src/weaviate_client_web/_framing.py @@ -19,9 +19,22 @@ _FLAG_TRAILER = 0x80 _FLAG_COMPRESSED = 0x01 +_KNOWN_FLAGS = _FLAG_TRAILER | _FLAG_COMPRESSED _HEADER = struct.Struct(">BI") # 1 flag byte + 4-byte big-endian length +class FrameError(ValueError): + """The body is not a well-formed grpc-web response.""" + + +class UnknownFrameFlagError(FrameError): + """A flag byte outside the grpc-web set: the body is not grpc-web framing (JSON, HTML, …).""" + + +class TruncatedFrameError(FrameError): + """The body ends before the length its frame header announces.""" + + def encode_message(payload: bytes) -> bytes: """Frame a single (uncompressed) protobuf payload for sending.""" return _HEADER.pack(0x00, len(payload)) + payload @@ -30,31 +43,41 @@ def encode_message(payload: bytes) -> bytes: def iter_frames(buf: bytes) -> Iterator[Tuple[int, bytes]]: """Yield ``(flag, payload)`` for each frame in a grpc-web response body.""" off, n = 0, len(buf) - while off + 5 <= n: - flag, length = _HEADER.unpack_from(buf, off) + while off < n: + # Validate the flag before the length so a text body ('{', '<') is reported as + # non-grpc-web rather than as a truncated frame with a garbage length. + flag = buf[off] + if flag & ~_KNOWN_FLAGS: + raise UnknownFrameFlagError(f"unknown grpc-web frame flag 0x{flag:02x} at byte {off}") + if off + 5 > n: + raise TruncatedFrameError(f"truncated grpc-web frame header at byte {off}") + _, length = _HEADER.unpack_from(buf, off) off += 5 if off + length > n: - raise ValueError("truncated grpc-web frame") + raise TruncatedFrameError( + f"truncated grpc-web frame: header announces {length} bytes, {n - off} remain" + ) yield flag, buf[off : off + length] off += length - if off != n: - raise ValueError("trailing bytes after final grpc-web frame") def parse_trailers(raw: bytes) -> Dict[str, str]: """Parse a trailer frame payload into a lower-cased header dict. - Names are ASCII by spec, but values are decoded leniently: a proxy that does not - percent-encode ``grpc-message``, or a server error quoting a UTF-8 collection / - tenant / property name, puts raw non-ASCII bytes in the trailer. Failing here would - throw away the ``grpc-status`` that came with it. + Decoded leniently on both sides of the colon: a proxy that does not percent-encode + ``grpc-message``, or a server error quoting a UTF-8 collection / tenant / property + name, puts raw non-ASCII bytes in the trailer, and one odd key must not discard the + ``grpc-status`` travelling with it. Lines are CRLF-terminated by spec; bare LF is + accepted. """ out: Dict[str, str] = {} - for line in raw.split(b"\r\n"): + for line in raw.split(b"\n"): + line = line.rstrip(b"\r") if not line: continue key, _, value = line.partition(b":") - out[key.strip().decode("ascii").lower()] = value.strip().decode("utf-8", "replace") + name = key.strip().decode("utf-8", "replace").lower() + out[name] = value.strip().decode("utf-8", "replace") return out @@ -62,13 +85,17 @@ def split_response(body: bytes) -> Tuple[List[bytes], Dict[str, str]]: """Split a grpc-web response body into message payloads and trailers.""" messages: List[bytes] = [] trailers: Dict[str, str] = {} + seen_trailer = False for flag, payload in iter_frames(body): if flag & _FLAG_TRAILER: trailers.update(parse_trailers(payload)) + seen_trailer = True elif flag & _FLAG_COMPRESSED: - raise ValueError( + raise FrameError( "compressed grpc-web message frames are not supported by this transport" ) + elif seen_trailer: + raise FrameError("message frame after the trailer frame") else: messages.append(payload) return messages, trailers diff --git a/packages/web/src/weaviate_client_web/_shim.py b/packages/web/src/weaviate_client_web/_shim.py index dbf0d79c6..b04b8837a 100644 --- a/packages/web/src/weaviate_client_web/_shim.py +++ b/packages/web/src/weaviate_client_web/_shim.py @@ -8,14 +8,14 @@ 1. **Import surface** — every ``import grpc`` / ``from grpc(.aio) import ...`` executed while ``weaviate`` and its generated ``*_pb2_grpc`` stubs are imported - (``weaviate/config.py:4-5``, ``exceptions.py:7-8``, ``retry.py:5-6``, - ``connect/base.py:5-8``, ``connect/v4.py:24,29-32``, and the v6300 stub's - ``grpc.__version__`` / ``grpc._utilities.first_version_is_lower`` version gate). + (``weaviate/config.py``, ``exceptions.py``, ``retry.py``, ``connect/base.py``, + ``connect/v4.py``, and the v6300 stub's ``grpc.__version__`` / + ``grpc._utilities.first_version_is_lower`` version gate). 2. **Runtime type contract** — :class:`AioChannel` becomes ``grpc.aio.Channel`` so the real grpc-web channel (which subclasses it) passes the - ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py`` (lines 722, - 1241); :class:`AioRpcError` is the error the client catches and inspects via - ``.code()`` / ``.details()`` (``exceptions.py:62-76``, ``retry.py:30-31``). + ``isinstance(..., grpc.aio.Channel)`` assertions in ``connect/v4.py``; + :class:`AioRpcError` is the error the client catches and inspects via ``.code()`` / + ``.details()`` (``exceptions.py``, ``retry.py``). """ import enum @@ -34,8 +34,8 @@ class StatusCode(enum.Enum): """Mirror of ``grpc.StatusCode``. ``value`` is the canonical ``(int, str)`` tuple, matching grpcio so ``code.value[0]`` - / ``code.value[1]`` (``exceptions.py:63,66``) and ``code.name`` - (``connect/v4.py:1189``) behave identically. + / ``code.value[1]`` (``exceptions.py``) and ``code.name`` (``connect/v4.py``) behave + identically. """ OK = (0, "ok") @@ -122,11 +122,11 @@ def debug_error_string(self) -> Optional[str]: class StreamStreamCall: - """Stand-in for ``grpc.aio.StreamStreamCall`` (imported as a type at ``v4.py:31``).""" + """Stand-in for ``grpc.aio.StreamStreamCall`` (imported as a type by ``connect/v4.py``).""" class ChannelCredentials: - """Stand-in for ``grpc.ChannelCredentials`` (imported by ``config.py:4``).""" + """Stand-in for ``grpc.ChannelCredentials`` (imported by ``config.py``).""" def ssl_channel_credentials(*_args: Any, **_kwargs: Any) -> ChannelCredentials: @@ -152,8 +152,8 @@ class AioChannel: def first_version_is_lower(_version: str, _other: str) -> bool: """Stand-in for ``grpc._utilities.first_version_is_lower``. - Returning ``False`` makes the v6300 stub's version gate - (``weaviate_pb2_grpc.py:17-29``) pass. + Returning ``False`` makes the v6300 stub's import-time version gate + (``weaviate_pb2_grpc.py``) pass. """ return False diff --git a/packages/web/tests/test_framing.py b/packages/web/tests/test_framing.py index 5bcefe354..fd18a35a3 100644 --- a/packages/web/tests/test_framing.py +++ b/packages/web/tests/test_framing.py @@ -3,6 +3,9 @@ import pytest from weaviate_client_web._framing import ( + FrameError, + TruncatedFrameError, + UnknownFrameFlagError, encode_message, iter_frames, parse_trailers, @@ -29,12 +32,20 @@ def test_split_response_message_and_trailer(): def test_split_response_multiple_messages(): + # the splitter returns every message frame; whether more than one is acceptable is + # the channel's decision (a unary RPC rejects it) body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) messages, trailers = split_response(body) assert messages == [b"a", b"bb"] assert trailers["grpc-status"] == "0" +def test_split_response_message_after_trailer_raises(): + body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") + with pytest.raises(FrameError, match="after the trailer"): + split_response(body) + + def test_split_response_trailers_only(): body = _frame(b"grpc-status:7\r\ngrpc-message:denied\r\n", 0x80) messages, trailers = split_response(body) @@ -70,13 +81,40 @@ def test_split_response_survives_non_ascii_trailer(): assert trailers["grpc-status"] == "7" +def test_parse_trailers_accepts_lf_only_lines(): + parsed = parse_trailers(b"grpc-status:0\ngrpc-message:ok\n") + assert parsed == {"grpc-status": "0", "grpc-message": "ok"} + + +def test_parse_trailers_keeps_status_when_a_key_is_not_ascii(): + # one odd key from a proxy must not throw away the whole block + parsed = parse_trailers("x-caf\u00e9:1\r\ngrpc-status:0\r\n".encode("utf-8")) + assert parsed["grpc-status"] == "0" + assert parsed["x-caf\u00e9"] == "1" + + def test_truncated_frame_raises(): framed = encode_message(b"hello")[:-2] - with pytest.raises(ValueError): + with pytest.raises(TruncatedFrameError): list(iter_frames(framed)) + with pytest.raises(TruncatedFrameError): + list(iter_frames(b"\x00\x00\x00")) # shorter than one frame header + + +@pytest.mark.parametrize("first_byte", [b"{", b"<", b"\x02", b"\x40", b"\xff"]) +def test_unknown_frame_flag_raises(first_byte): + # a JSON / HTML body, or a flag bit this transport does not know + body = first_byte + b"\x00\x00\x00\x01x" + with pytest.raises(UnknownFrameFlagError, match="unknown grpc-web frame flag"): + list(iter_frames(body)) + + +def test_frame_errors_are_value_errors(): + assert issubclass(TruncatedFrameError, ValueError) + assert issubclass(UnknownFrameFlagError, ValueError) def test_compressed_message_frame_rejected(): body = _frame(b"x", 0x01) - with pytest.raises(ValueError): + with pytest.raises(FrameError, match="compressed"): split_response(body) diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index 3c7d30dcc..02871fa69 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -7,11 +7,17 @@ import asyncio import struct +import sys from typing import Dict, List, Optional, Tuple import pytest -from weaviate_client_web._channel import GrpcWebChannel, set_sender +from weaviate_client_web._channel import ( + GrpcWebChannel, + _body_excerpt, + _encode_timeout, + set_sender, +) from weaviate_client_web._shim import AioChannel, AioRpcError, StatusCode @@ -80,7 +86,7 @@ def test_health_call_without_metadata(): sender = FakeSender(body=_ok_response(b"pong")) channel = _channel(sender) mc = channel.unary_unary("/grpc.health.v1.Health/Check", lambda x: x, lambda b: b) - # mirrors connect/v4.py:316 — request + timeout, no metadata + # mirrors the health check in connect/v4.py — request + timeout, no metadata assert asyncio.run(mc(b"ping", timeout=2)) == b"pong" @@ -118,9 +124,8 @@ def test_trailers_only_status_in_http_headers(): # --- non-grpc-web responses ------------------------------------------------------- # # Every real error response carries a body, and none of them is grpc-web framing. The -# bodies below are verbatim shapes seen in the wild; an empty error body (the old -# fixture) is the one shape no server or proxy produces, which is why these tests used -# to pass against a code path that never ran. +# bodies below are verbatim shapes seen in the wild (an empty error body is the one +# shape no server or proxy produces). # Weaviate's own 404, verbatim from a 1.39.0 server asked for the wrong prefix. WEAVIATE_404_JSON = ( @@ -192,6 +197,61 @@ def test_nginx_404_html_is_reported_as_an_http_404(): assert "malformed grpc-web response" not in err.details() +def test_405_names_the_wrong_prefix(): + # a 405 can only come from an existing HTTP route (measured live: a prefix pointing + # at /v1/objects answers "method POST is not allowed"), so the prefix is wrong + err = _details_of(405, b'{"code":405,"message":"method POST is not allowed, but [GET] are"}') + assert err.details().startswith("HTTP 405 ") + assert "path prefix" in err.details() + assert "/v1/grpc-web" in err.details() + assert "method POST is not allowed" in err.details() + + +def test_truncated_grpc_web_body_is_reported_as_truncated_not_as_wrong_prefix(): + # a valid frame header whose payload was cut short: the endpoint IS grpc-web, so the + # SPA / path-prefix hint would send the user the wrong way + body = _ok_response(b"reply-bytes")[:-6] + err = _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "truncated" in err.details() + assert "cut short" in err.details() + assert "single-page-app" not in err.details() + assert "path prefix" not in err.details() + + +def test_spa_html_body_is_not_reported_as_truncated(): + # text bodies decode to an unknown flag byte and a garbage length; they must read as + # "not grpc-web framing", never as a truncated grpc-web body + err = _details_of(200, SPA_INDEX_HTML) + assert "truncated" not in err.details() + assert "not grpc-web framing" in err.details() + + +def test_message_frame_after_trailer_is_internal(): + body = _frame(b"a") + _frame(b"grpc-status:0\r\n", 0x80) + _frame(b"late") + err = _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "malformed grpc-web response" in err.details() + assert "after the trailer" in err.details() + assert "single-page-app" not in err.details() + + +def test_multiple_message_frames_in_unary_response_is_internal(): + # a unary RPC has exactly one message; silently taking the first would hide a proxy + # or server that streams several + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:0\r\n", 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.INTERNAL + assert "2 message frames" in err.details() + + +def test_error_status_wins_over_multiple_message_frames(): + body = _frame(b"a") + _frame(b"bb") + _frame(b"grpc-status:5\r\ngrpc-message:gone\r\n", 0x80) + err = _details_of(200, body) + assert err.code() is StatusCode.NOT_FOUND + assert err.details() == "gone" + + def test_spa_fallback_html_200_is_distinguishable_from_a_404(): # An HTTP 200 serving index.html is the other half of a wrong path prefix: the app's # catch-all route answers instead of Weaviate. It must not read as malformed framing. @@ -396,6 +456,110 @@ def test_grpc_timeout_header_rounds_up(): assert sender.calls[0][1]["grpc-timeout"] == "124m" +def test_body_excerpt_empty_and_non_printable(): + assert _body_excerpt(b"") == "" + assert _body_excerpt(b"\x00\x01\x02\x7f") == "<4 non-printable bytes>" + assert _body_excerpt(b"ok\x00\x01") == "ok" + + +@pytest.mark.parametrize( + "seconds,expected", + [ + (None, None), + (float("inf"), None), + (float("nan"), None), + (0, "1m"), + (0.1234, "124m"), + (5, "5000m"), + (99_999, "99999000m"), + (100_000, "100000S"), # 1e8 ms would be 9 digits + (1e8, "1666667M"), # 1e8 s would be 9 digits + (1e9, "16666667M"), + (5_999_999_940, "99999999M"), # the largest deadline that still fits in minutes + (1e10, None), # would need hours, which transcoders reject above 8H: no deadline + (1e15, None), + ], +) +def test_encode_timeout_stays_within_eight_digits(seconds, expected): + encoded = _encode_timeout(seconds) + assert encoded == expected + if encoded is not None: + assert len(encoded) <= 9 # 8 digits + unit + assert not encoded.endswith("H") + + +def test_infinite_timeout_sends_no_deadline(): + # Timeout(query=inf): neither a grpc-timeout header nor a client-side wait + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + assert asyncio.run(mc(b"q", timeout=float("inf"))) == b"x" + _, headers, _, timeout = sender.calls[0] + assert "grpc-timeout" not in headers + assert timeout is None + + +def test_huge_timeout_uses_minutes_then_no_deadline(): + # 1e8 s in milliseconds is 12 digits; the server rejects more than 8 ("timeout is + # too long", HTTP 400) and transcoders reject hour values above 8H, so past the + # minute range the request carries no deadline at all + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + asyncio.run(mc(b"q", timeout=1e8)) + asyncio.run(mc(b"q", timeout=1e9)) + asyncio.run(mc(b"q", timeout=1e10)) + assert sender.calls[0][1]["grpc-timeout"] == "1666667M" + assert sender.calls[1][1]["grpc-timeout"] == "16666667M" + assert "grpc-timeout" not in sender.calls[2][1] + assert sender.calls[2][3] is None # no client-side wait either + + +@pytest.mark.parametrize("bad", ["val\r\nx-injected: evil", "val\nx", "v\0"]) +def test_crlf_in_metadata_rejected(bad): + sender = FakeSender(body=_ok_response(b"x")) + channel = _channel(sender) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(ValueError, match="Illegal character"): + asyncio.run(mc(b"q", metadata=[("x-key", bad)])) + with pytest.raises(ValueError, match="Illegal character"): + asyncio.run(mc(b"q", metadata=[("x-key\r\n", "v")])) + assert sender.calls == [] + + +def _unavailable_details(monkeypatch, path_prefix, platform): + async def boom(url, headers, body, timeout): + raise ConnectionError("Failed to fetch") + + monkeypatch.setattr(sys, "platform", platform) + channel = GrpcWebChannel("h:50051", secure=False, sender=boom, path_prefix=path_prefix) + mc = channel.unary_unary("/svc/M", lambda x: x, lambda b: b) + with pytest.raises(AioRpcError) as excinfo: + asyncio.run(mc(b"q")) + assert excinfo.value.code() is StatusCode.UNAVAILABLE + return excinfo.value.details() + + +def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(monkeypatch): + # use_async_with_local() under Pyodide builds a prefix-less channel aimed at the + # native gRPC port, which fetch cannot reach; the error must say what to do instead + details = _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") + assert "grpc_path_prefix='/v1/grpc-web'" in details + assert "1.38.3" in details + assert "use_async_with_custom" in details + + +def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): + details = _unavailable_details(monkeypatch, path_prefix="/v1/grpc-web", platform="emscripten") + assert "no grpc_path_prefix" not in details + + +def test_unavailable_without_path_prefix_off_emscripten_has_no_prefix_hint(monkeypatch): + # on CPython an empty prefix against a transcoder is the normal configuration + details = _unavailable_details(monkeypatch, path_prefix="", platform="linux") + assert "no grpc_path_prefix" not in details + + def test_close_is_awaitable_noop(): channel = _channel(FakeSender()) assert asyncio.run(channel.close()) is None From 1ffa2a0500c35c7b6bfa0d2161149f659e22c9ef Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:34:35 +0200 Subject: [PATCH 23/27] docs(web): describe the core-native grpc-web setup first use_async_with_custom(..., grpc_path_prefix="/v1/grpc-web") on the REST port is the primary example; use_async_with_local/weaviate_cloud take no prefix and only work with a transcoder at the gRPC host:port root. Drop the skip_init_checks recommendation (the health check runs over grpc-web), list every header the client sends in the CORS section, state the tested Pyodide version, and correct the message-size and REST-transport wording. --- packages/web/README.md | 77 ++++++++++++++----- .../web/src/weaviate_client_web/__init__.py | 16 +++- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/packages/web/README.md b/packages/web/README.md index 800f476ae..e1aa554f1 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -8,6 +8,10 @@ WASM workers) where there is no socket and no `grpcio` wheel. It is built from the same repository as `weaviate-client` and reuses its generated protobuf stubs — it does **not** fork code generation. +Requires Weaviate ≥ 1.38.3 (the first release to serve grpc-web natively) or a grpc-web +transcoder in front of an older server. Pyodide ≥ 0.27 recommended; verified on +Pyodide 314.0.4 (CPython 3.14). + ## How it works Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard-imports @@ -23,31 +27,55 @@ Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard- (`grpc.__version__` / `grpc._utilities.first_version_is_lower`). The `GrpcWebChannel` frames unary RPCs as grpc-web (a 5-byte header + protobuf payload) -and POSTs them via `pyodide.http.pyfetch` to a server fronted by a grpc-web transcoder -(e.g. Envoy or [connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)). Call -metadata (API key / OIDC bearer) is folded into `fetch` headers. - -For REST, Pyodide ≥ 0.27 distributes a patched httpx that already routes through the -browser's `fetch` natively — when that build is detected the package leaves it alone. -Only when httpx resolved from PyPI (httpcore + raw sockets, which cannot work under -WASM) does the package patch `httpx.AsyncHTTPTransport` with its own pyfetch-based -transport. +and POSTs them via `pyodide.http.pyfetch`. Call metadata (API key / OIDC bearer) is +folded into `fetch` headers. Two deployments work: + +1. **Core-native grpc-web (Weaviate ≥ 1.38.3).** Weaviate serves grpc-web on its REST + port under `/v1/grpc-web`, so gRPC and REST share one host, port and TLS setup and + no proxy is needed. Select it with `grpc_path_prefix="/v1/grpc-web"` and point + `grpc_host`/`grpc_port` at the REST endpoint. +2. **A grpc-web transcoder** (e.g. Envoy or + [connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)) listening at the + root of `grpc_host:grpc_port` in front of Weaviate's native gRPC port. This is what + `use_async_with_local()` / `use_async_with_weaviate_cloud()` need, because those + helpers have no `grpc_path_prefix` parameter. + +For REST (`is_ready`, collection config, `/batch/references`, …) the package patches +`httpx.AsyncHTTPTransport` with its own `pyfetch`-based transport. It does so even on +Pyodide builds whose bundled httpx has a JS-fetch transport of its own: that transport +cannot read the null body of HEAD requests and 204 responses (`data.exists`, +`data.delete_by_id`, `tenants.exists`, …) and does not enforce the client's per-request +timeouts. ## Usage With this package installed, a plain `import weaviate` is all you need — under Emscripten the base client imports `weaviate_client_web` itself before anything else, -which installs the shim (and raises a clear error if the package is missing): +which installs the shim and the fetch transport (and raises a clear error if the package +is missing). Against Weaviate ≥ 1.38.3: ```python import weaviate # bootstraps weaviate_client_web automatically under Emscripten -client = weaviate.use_async_with_local(skip_init_checks=True) -await client.connect() +client = weaviate.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", # same host as REST + grpc_port=8080, # same port as REST: grpc-web rides on the REST listener + grpc_secure=False, + grpc_path_prefix="/v1/grpc-web", +) +await client.connect() # runs the gRPC health check over grpc-web collection = client.collections.get("Article") await collection.query.near_text("hello", limit=3) ``` +`use_async_with_local()` and `use_async_with_weaviate_cloud()` have no +`grpc_path_prefix`, so under Pyodide they only work when a grpc-web transcoder answers at +the root of `grpc_host:grpc_port`. Pass `headers={...}` / `auth_credentials=...` to +`use_async_with_custom` as usual for API keys, OIDC or WCD. + Importing the companion explicitly first also works and remains the explicit form: ```python @@ -60,8 +88,8 @@ import weaviate | Feature | Kind | Status | |----------------------------------------------------------|-----------------|--------| | Search, Aggregate, TenantsGet, BatchObjects, BatchDelete | unary gRPC | ✅ works over grpc-web | -| Health check (`/grpc.health.v1.Health/Check`) | unary gRPC | ✅ (recommend `skip_init_checks=True` + REST `/.well-known/ready`) | -| REST (`is_ready`, config, `/batch/references`, …) | REST | ✅ via fetch (Pyodide's httpx build, or this package's fallback transport) | +| Health check (`/grpc.health.v1.Health/Check`) | unary gRPC | ✅ runs on `connect()` over grpc-web | +| REST (`is_ready`, config, `/batch/references`, …) | REST | ✅ via the package's own fetch transport | | API-key auth (`Auth.api_key`) | header | ✅ | | OIDC auth (`client_credentials` / `client_password` / `bearer_token`) | REST | ✅ token fetch + asyncio-task refresh (no threads) | | Bulk insert: `collection.data.insert_many()` | unary gRPC | ✅ the supported bulk path under WASM | @@ -70,7 +98,7 @@ import weaviate | Embedded Weaviate (`use_async_with_embedded`) | subprocess | ❌ raises "not supported under WebAssembly/Pyodide" | | Synchronous client | — | ❌ async-only under WASM | | Weaviate Agents: `AsyncQueryAgent` `run/ask/search` | REST | ✅ via fetch | -| Weaviate Agents: `ask_stream` / `research_stream` (SSE) | REST streaming | ⚠️ degraded under the fallback transport: fully buffered, events arrive only when the run completes (and long runs can hit the request timeout) | +| Weaviate Agents: `ask_stream` / `research_stream` (SSE) | REST streaming | ⚠️ degraded: the fetch transport buffers the whole response, so events arrive only when the run completes (and long runs can hit the request timeout) | | Weaviate Agents: sync `QueryAgent`, `TransformationAgent`, `PersonalizationAgent` | REST sync | ❌ no async flavour exists | ## Configuration not honored in the browser @@ -82,17 +110,22 @@ under WASM: cannot proxy fetch requests per-client), - connection-pool sizing and `session_pool_max_retries`, - `GrpcConfig.credentials` (custom CA bundles — the browser's trust store decides TLS), -- `GrpcConfig.channel_options`, including `grpc.max_send/receive_message_length` - (only `grpc-web.path_prefix` is consumed), +- `GrpcConfig.channel_options`, including `grpc.max_send_message_length` / + `grpc.max_receive_message_length` (only `grpc-web.path_prefix` is consumed). The + practical message-size ceiling is the server's `grpcMaxMessageSize` (reported by + `/v1/meta`); exceeding it surfaces as `RESOURCE_EXHAUSTED`, - `Proxies.grpc` / `GRPC_PROXY`. ## CORS requirements (browsers) -Cross-origin browser deployments must configure the grpc-web transcoder / REST endpoint -with CORS, or failures become hard to diagnose: +Weaviate ≥ 1.38.3 serves the CORS headers below for its `/v1/grpc-web` endpoint itself, +with no configuration. Its request-header list is a **closed allowlist**: custom +`headers={...}` that are not on it fail the browser's preflight. Cross-origin +deployments that go through a grpc-web transcoder or a proxy must configure CORS there: -- allow the request headers the client sends: `authorization`, `content-type`, - `x-grpc-web`, and any custom headers; +- allow every request header the client sends: `content-type`, `x-grpc-web`, + `x-user-agent`, `grpc-timeout`, `x-weaviate-client`, `authorization` (when auth is + used) and `x-weaviate-cluster-url` (Weaviate Cloud); - expose the grpc-web status headers on responses: `Access-Control-Expose-Headers: grpc-status, grpc-message` — without this, trailers-only error responses (e.g. a bad API key) are reported as @@ -106,3 +139,5 @@ with CORS, or failures become hard to diagnose: interpreter (run it in a fresh process, before importing `weaviate`). Inject a sender with `weaviate_client_web.set_sender(...)` (e.g. `make_httpx_sender()`) to exercise the transport against an Envoy/vanguard transcoder without a browser. +`install_fetch_transport(force=True)` likewise patches httpx on CPython, given an +importable `pyodide.http` stand-in. diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py index f0a481d19..24820d41d 100644 --- a/packages/web/src/weaviate_client_web/__init__.py +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -3,16 +3,26 @@ Under Pyodide/Emscripten there is no ``grpcio`` wheel. Importing this package installs a pure-Python ``grpc`` shim into ``sys.modules`` (and forces the pure-Python protobuf runtime) so that the subsequent ``import weaviate`` succeeds and its async gRPC data path -runs over grpc-web (``fetch``) instead of HTTP/2 sockets. +runs over grpc-web (``fetch``) instead of HTTP/2 sockets; REST runs through the package's +own ``fetch``-based httpx transport. -Usage under Pyodide (with this package installed, a bare ``import weaviate`` suffices — +Usage under Pyodide against Weaviate >= 1.38.3, which serves grpc-web on its REST port +under ``/v1/grpc-web`` (with this package installed, a bare ``import weaviate`` suffices — the base client imports this package itself under Emscripten before anything else):: import weaviate - client = weaviate.use_async_with_local(skip_init_checks=True) + client = weaviate.use_async_with_custom( + http_host="localhost", http_port=8080, http_secure=False, + grpc_host="localhost", grpc_port=8080, grpc_secure=False, + grpc_path_prefix="/v1/grpc-web", + ) await client.connect() +``use_async_with_local`` / ``use_async_with_weaviate_cloud`` have no ``grpc_path_prefix`` +and therefore only work with a grpc-web transcoder (Envoy, vanguard) at the root of +``grpc_host:grpc_port``. + An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and remains the explicit form. The shim is installed automatically only under Emscripten, so importing this package on a normal CPython install never clobbers a real, working From b950e39ae6b0fdf0d47a9b1c0720d4c3fe597b17 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:34:54 +0200 Subject: [PATCH 24/27] fix(connect): back off token refresh, await the refresher on close, sanitise timeouts A permanent OAuth failure (expired or revoked refresh token) made the async refresher POST to the IdP every second; the sync thread died silently on the same error. Both colours now catch any exception and retry with a capped exponential backoff (1s doubling to 60s, reset on success); Con001 says so. The async close() awaits the cancelled task, and the sync thread waits on the Event it was started with, so close() ends it promptly and a reconnect leaves no second refresher. A test drives a real refresher death through connect() to pin the done-callback wiring. Non-finite timeouts mean "no deadline" for every REST and gRPC hand-off, the Emscripten connect-timeout arm is gone (the package transport reads only the read timeout), the grpc-web prefix guards run at construction with a message that explains the CPython testing mode, and a RuntimeError is rewritten to WeaviateClosedClientError only when the http client is actually closed. Native connection errors keep the observed gRPC status line; the duplicated no-prefix hint lives in the transport only. --- mock_tests/test_auth.py | 220 ++++++++++++++++++++++++++---- test/test_connection.py | 90 ++++++++++++ test/test_connection_params.py | 48 +++++++ test/test_wasm_compat.py | 81 ++++++++--- weaviate/connect/base.py | 42 +++--- weaviate/connect/helpers.py | 21 +-- weaviate/connect/v4.py | 241 +++++++++++++++++++++------------ weaviate/warnings.py | 10 +- 8 files changed, 591 insertions(+), 162 deletions(-) create mode 100644 test/test_connection.py diff --git a/mock_tests/test_auth.py b/mock_tests/test_auth.py index 95d64858e..cbd5e1f4a 100644 --- a/mock_tests/test_auth.py +++ b/mock_tests/test_auth.py @@ -1,8 +1,9 @@ import asyncio import json +import threading import time import warnings -from typing import Union +from typing import List, Union import grpc import pytest @@ -12,7 +13,7 @@ import weaviate from mock_tests.conftest import CLIENT_ID, MOCK_IP, MOCK_PORT, MOCK_PORT_GRPC from weaviate.connect.v4 import _ConnectionBase -from weaviate.exceptions import MissingScopeException +from weaviate.exceptions import MissingScopeException, UnexpectedStatusCodeError ACCESS_TOKEN = "HELLO!IamAnAccessToken" CLIENT_SECRET = "SomeSecret.DontTell" @@ -123,27 +124,48 @@ def handler(request: Request) -> Response: assert token_requests > first # a fresh token was fetched with the credentials -@pytest.mark.asyncio -async def test_token_refresh_survives_non_http_error_async( - weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server, recwarn -) -> None: - """A refresh failure that is NOT an httpx.HTTPError must warn and keep the refresher alive. +def _reject_refreshes(weaviate_auth_mock: HTTPServer) -> List[float]: + """Make the IdP reject every refresh (400 invalid_grant); returns the hit timestamps.""" + hits: List[float] = [] - An IdP rejecting the refresh (400 invalid_grant) makes authlib raise OAuthError. When - only HTTPError was caught, the refresh task died silently — nothing awaits it, so not - even asyncio's 'exception was never retrieved' fired — and every later request 401ed - with nothing pointing at the token refresh. - """ - weaviate_auth_mock.expect_request("/auth").respond_with_response( - Response( + def handler(request: Request) -> Response: + hits.append(time.monotonic()) + return Response( json.dumps({"error": "invalid_grant", "error_description": "refresh token expired"}), status=400, content_type="application/json", ) - ) + + weaviate_auth_mock.expect_request("/auth").respond_with_handler(handler) weaviate_auth_mock.expect_request( "/v1/schema", headers={"Authorization": "Bearer " + ACCESS_TOKEN} ).respond_with_json({"classes": []}) + return hits + + +def _assert_backed_off(hits: List[float], recwarn) -> None: + # attempts at ~1s, ~2s, ~4s after connect: a fixed 1s retry would have made 5 in 5s + assert 2 <= len(hits) <= 3 + gaps = [b - a for a, b in zip(hits, hits[1:])] + assert all(later > earlier * 1.5 for earlier, later in zip(gaps, gaps[1:])) + failed = [w for w in recwarn if str(w.message).startswith("Con001")] + assert len(failed) == len(hits) # one warning per failed attempt + assert "invalid_grant" in str(failed[0].message) + assert "retrying in 1s" in str(failed[0].message) + assert "retrying in 2s" in str(failed[1].message) + assert "unstable internet" not in str(failed[0].message) + + +@pytest.mark.asyncio +async def test_token_refresh_backs_off_on_persistent_failure_async( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server, recwarn +) -> None: + """A permanently failing refresh must neither kill the refresher nor hot-loop the IdP. + + A 400 invalid_grant makes authlib raise OAuthError: warn, back off exponentially, + stay alive. + """ + hits = _reject_refreshes(weaviate_auth_mock) async with weaviate.use_async_with_local( host=MOCK_IP, @@ -157,23 +179,132 @@ async def test_token_refresh_survives_non_http_error_async( ) as client: task = getattr(client._connection, "_ConnectionBase__token_refresh_task") # noqa: B009 assert task is not None - await asyncio.sleep(2.5) # long enough for at least two failed attempts - assert not task.done() # the refresher survived the failure + await asyncio.sleep(5) + assert not task.done() # the refresher survived the failures await client.collections.list_all() # ... and the client still works - failed = [w for w in recwarn if str(w.message).startswith("Con001")] - assert len(failed) >= 1 - assert "invalid_grant" in str(failed[0].message) + _assert_backed_off(hits, recwarn) # the task ended by close()'s cancellation, not by dying on the exception assert [w for w in recwarn if str(w.message).startswith("Con003")] == [] +def test_token_refresh_backs_off_on_persistent_failure( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server, recwarn +) -> None: + """Sync colour of the test above. + + The daemon thread used to die silently on anything but an httpx.HTTPError; now it + warns and backs off like the async task. + """ + hits = _reject_refreshes(weaviate_auth_mock) + + threads_before = set(threading.enumerate()) + with weaviate.connect_to_local( + host=MOCK_IP, + port=MOCK_PORT, + grpc_port=MOCK_PORT_GRPC, + auth_credentials=weaviate.auth.AuthBearerToken( + ACCESS_TOKEN, refresh_token=REFRESH_TOKEN, expires_in=1 + ), + ) as client: + refreshers = [ + t for t in set(threading.enumerate()) - threads_before if t.name == "TokenRefresh" + ] + assert len(refreshers) == 1 + time.sleep(5) + assert refreshers[0].is_alive() # survived the failures + client.collections.list_all() + + _assert_backed_off(hits, recwarn) + refreshers[0].join(timeout=2) + assert not refreshers[0].is_alive() # close() stops the daemon thread promptly + + +class _Boom(BaseException): + """Escapes the refresh loop's `except Exception` like a real BaseException would.""" + + +@pytest.mark.asyncio +async def test_token_refresh_death_is_surfaced_through_real_connect( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server, recwarn +) -> None: + """A refresher started by connect() that dies outside the loop body must warn (Con003). + + Pins the done-callback wiring in _create_background_token_refresh, which the + callback-only unit test below cannot see. + """ + weaviate_auth_mock.expect_request( + "/v1/schema", headers={"Authorization": "Bearer " + ACCESS_TOKEN} + ).respond_with_json({"classes": []}) + + async def boom(*args, **kwargs): + raise _Boom("boom") + + async with weaviate.use_async_with_local( + host=MOCK_IP, + port=MOCK_PORT, + grpc_port=MOCK_PORT_GRPC, + auth_credentials=weaviate.auth.AuthBearerToken( + ACCESS_TOKEN, refresh_token=REFRESH_TOKEN, expires_in=1 + ), + ) as client: + task = getattr(client._connection, "_ConnectionBase__token_refresh_task") # noqa: B009 + assert task is not None + client._connection._client.refresh_token = boom # type: ignore[union-attr] + await asyncio.sleep(2) # the first refresh is due after ~1s + assert task.done() and not task.cancelled() + + stopped = [w for w in recwarn if str(w.message).startswith("Con003")] + assert len(stopped) == 1 + assert "_Boom" in str(stopped[0].message) + + +@pytest.mark.asyncio +async def test_failed_connect_cancels_the_refresher_and_close_is_safe_after( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server +) -> None: + """A failed connect() must not leave the refresher running; close() after it is safe. + + The OIDC step starts the refresher before /v1/meta is checked, and close() afterwards + (even twice) must neither hang nor raise. + """ + weaviate_auth_mock.expect_request("/auth").respond_with_json( + {"access_token": ACCESS_TOKEN, "expires_in": 500, "refresh_token": REFRESH_TOKEN} + ) + # oneshot handlers take precedence over the fixture's permanent /v1/meta handler + weaviate_auth_mock.expect_oneshot_request("/v1/meta").respond_with_response( + Response(status=500) + ) + + tasks_before = asyncio.all_tasks() + client = weaviate.use_async_with_local( + host=MOCK_IP, + port=MOCK_PORT, + grpc_port=MOCK_PORT_GRPC, + auth_credentials=weaviate.auth.AuthBearerToken( + ACCESS_TOKEN, refresh_token=REFRESH_TOKEN, expires_in=500 + ), + ) + with pytest.raises(UnexpectedStatusCodeError): + await client.connect() + + refresh_tasks = [ + t for t in asyncio.all_tasks() - tasks_before if "token_refresh" in repr(t.get_coro()) + ] + assert len(refresh_tasks) == 1 # it was started ... + await asyncio.sleep(0) + assert refresh_tasks[0].cancelled() # ... and cancelled by the failed connect + + await asyncio.wait_for(client.close(), timeout=2) + await asyncio.wait_for(client.close(), timeout=2) + + @pytest.mark.asyncio async def test_token_refresh_death_outside_loop_body_is_surfaced() -> None: """A refresher that dies where the loop body cannot catch it must still warn. - Nothing awaits the task and _cancel_background_token_refresh drops the reference, so - the done-callback is the only thing that can observe such a death. + Nothing awaits the task while it runs (close() only gathers it, exceptions included), + so the done-callback is the only thing that can observe such a death. """ on_done = getattr(_ConnectionBase, "_ConnectionBase__warn_if_token_refresh_died") # noqa: B009 @@ -347,11 +478,52 @@ async def test_async_auth_starts_no_threads( t for t in asyncio.all_tasks() - tasks_before if "token_refresh" in repr(t.get_coro()) ] assert len(refresh_tasks) == 1 # the refresher runs as an asyncio task instead - # ... and close() must cancel it, not leak it (one wait for the cancellation to land) - await asyncio.wait(refresh_tasks, timeout=1) + # ... and close() must cancel it AND await it: done as soon as close() returns assert refresh_tasks[0].done() +def test_sync_reconnect_leaves_exactly_one_refresher_thread( + weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server +) -> None: + """close() must end the daemon thread promptly, and a later connect() must not revive it. + + The thread used to re-read the connection's shutdown event on every loop, so after + close()+connect() it picked up the NEW (unset) event and kept refreshing next to the + new thread — two refreshers per client. + """ + weaviate_auth_mock.expect_request("/auth").respond_with_json( + {"access_token": ACCESS_TOKEN, "expires_in": 500, "refresh_token": REFRESH_TOKEN} + ) + weaviate_auth_mock.expect_request( + "/v1/schema", headers={"Authorization": "Bearer " + ACCESS_TOKEN} + ).respond_with_json({"classes": []}) + + def refreshers() -> List[threading.Thread]: + return [t for t in set(threading.enumerate()) - threads_before if t.name == "TokenRefresh"] + + threads_before = set(threading.enumerate()) + client = weaviate.connect_to_local( + host=MOCK_IP, + port=MOCK_PORT, + grpc_port=MOCK_PORT_GRPC, + auth_credentials=weaviate.auth.AuthBearerToken( + ACCESS_TOKEN, refresh_token=REFRESH_TOKEN, expires_in=500 + ), + ) + (first,) = refreshers() + client.close() + first.join(timeout=2) + assert not first.is_alive() # not asleep until the next (470s away) wake-up + + client.connect() + client.collections.list_all() + alive = [t for t in refreshers() if t.is_alive()] + assert len(alive) == 1 and alive[0] is not first + client.close() + alive[0].join(timeout=2) + assert not alive[0].is_alive() + + def test_refresh_of_refresh(weaviate_auth_mock: HTTPServer, start_grpc_server: grpc.Server) -> None: """Test that refresh tokens are used to get a new refresh token token.""" weaviate_auth_mock.expect_request( diff --git a/test/test_connection.py b/test/test_connection.py new file mode 100644 index 000000000..28540f162 --- /dev/null +++ b/test/test_connection.py @@ -0,0 +1,90 @@ +"""Unit tests for weaviate.connect.v4 connection-level behaviour that needs no server.""" + +import asyncio +import inspect +import threading +import time + +import grpc +import pytest +from grpc.aio import AioRpcError, Metadata + +from weaviate.config import ConnectionConfig +from weaviate.config import Timeout as TimeoutConfig +from weaviate.connect.base import ConnectionParams +from weaviate.connect.v4 import ConnectionAsync, ConnectionSync, _ConnectionBase +from weaviate.exceptions import WeaviateBatchError +from weaviate.proto.v1 import batch_pb2 + + +def test_connection_sync_init_mirrors_the_base_signature() -> None: + # explicit, typed parameters (no *args/**kwargs pass-through) + assert ( + inspect.signature(ConnectionSync.__init__).parameters + == inspect.signature(_ConnectionBase.__init__).parameters + ) + + +def _connection_async() -> ConnectionAsync: + return ConnectionAsync( + connection_params=ConnectionParams.from_url("http://localhost:8080", 50051), + auth_client_secret=None, + timeout_config=TimeoutConfig(), + proxies=None, + trust_env=False, + additional_headers=None, + connection_config=ConnectionConfig(), + ) + + +def test_async_batch_objects_error_carries_the_grpc_details_only() -> None: + # like WeaviateQueryError: the message is the server's details, not the whole + # '' repr + class FailingStub: + async def BatchObjects(self, request, metadata=None, timeout=None): + raise AioRpcError( + grpc.StatusCode.INVALID_ARGUMENT, Metadata(), Metadata(), details="bad object" + ) + + conn = _connection_async() + conn._connected = True + conn._grpc_stub = FailingStub() # type: ignore[assignment] + + with pytest.raises(WeaviateBatchError) as excinfo: + asyncio.run( + conn.grpc_batch_objects(batch_pb2.BatchObjectsRequest(), timeout=1, max_retries=0) + ) + assert excinfo.value.message == "bad object" + assert "AioRpcError" not in str(excinfo.value) + + +def test_cancel_refresher_owned_by_another_loop_is_scheduled_not_awaited() -> None: + # close() from a thread/loop other than the refresher's must not try to await the + # task (which would hang); it schedules the cancel on the owning loop instead + loop = asyncio.new_event_loop() + started = threading.Event() + holder: dict = {} + + async def forever() -> None: + started.set() + await asyncio.sleep(3600) + + def run() -> None: + holder["task"] = loop.create_task(forever()) + loop.run_forever() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + assert started.wait(timeout=2) + conn = _connection_async() + conn._ConnectionBase__token_refresh_task = holder["task"] # type: ignore[attr-defined] + try: + assert conn._cancel_background_token_refresh() is None # nothing to await here + deadline = time.monotonic() + 2 + while not holder["task"].done() and time.monotonic() < deadline: + time.sleep(0.01) + assert holder["task"].cancelled() + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=2) + loop.close() diff --git a/test/test_connection_params.py b/test/test_connection_params.py index 89b9a4cf4..a0dc1c7d4 100644 --- a/test/test_connection_params.py +++ b/test/test_connection_params.py @@ -171,3 +171,51 @@ def fake_insecure_channel(target, options=None, **kwargs): option_keys = [key for key, _ in captured["options"]] assert "grpc-web.path_prefix" not in option_keys + + +def test_connect_to_custom_treats_root_prefix_as_native_grpc(monkeypatch) -> None: + # "/" normalizes to "" (native gRPC), so the async-only guard must not fire on it + import weaviate + import weaviate.connect.helpers as helpers + + monkeypatch.setattr(helpers, "__connect", lambda client: client) # module-level: not mangled + client = weaviate.connect_to_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=50051, + grpc_secure=False, + grpc_path_prefix="/", + ) + assert client._connection._connection_params._grpc_web_path_prefix == "" + + +def test_async_client_construction_rejects_prefix_without_shim(monkeypatch) -> None: + # fail at construction with actionable text, not deep inside connect() after the + # OIDC and /v1/meta round trips already succeeded + from weaviate import WeaviateAsyncClient + + monkeypatch.delattr(base_mod.grpc, "__weaviate_client_web_shim__", raising=False) + with pytest.raises(WeaviateInvalidInputError) as excinfo: + WeaviateAsyncClient(_grpc_web_params()) + msg = str(excinfo.value) + assert "weaviate-client-web" in msg + assert "install(force=True)" in msg + assert "set_sender(make_httpx_sender())" in msg + assert "import weaviate" in msg # ... and how Pyodide differs + + +def test_async_client_construction_allows_prefix_with_shim(monkeypatch) -> None: + from weaviate import WeaviateAsyncClient + + monkeypatch.setattr(base_mod.grpc, "__weaviate_client_web_shim__", True, raising=False) + client = WeaviateAsyncClient(_grpc_web_params()) + assert client._connection._connection_params._grpc_web_path_prefix == "/grpc-web" + + +def test_sync_client_construction_rejects_grpc_web_prefix() -> None: + from weaviate import WeaviateClient + + with pytest.raises(WeaviateInvalidInputError, match="async"): + WeaviateClient(_grpc_web_params()) diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index f568d08e9..5ac1d58fc 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -6,8 +6,10 @@ """ import sys +from typing import Optional import grpc +import httpx import pytest from grpc.aio import AioRpcError, Metadata from httpx import ConnectError, ConnectTimeout, PoolTimeout, ReadTimeout, WriteTimeout @@ -16,7 +18,7 @@ from weaviate.config import ConnectionConfig from weaviate.config import Timeout as TimeoutConfig from weaviate.connect.base import ConnectionParams -from weaviate.connect.v4 import _ConnectionBase, _exc_detail +from weaviate.connect.v4 import _ConnectionBase, _deadline, _exc_detail from weaviate.embedded import _EmbeddedBase from weaviate.exceptions import ( WeaviateClosedClientError, @@ -58,25 +60,33 @@ def test_async_client_construction_allowed_under_emscripten(monkeypatch) -> None assert client is not None -def _handle_exceptions(e: Exception, error_msg: str = "") -> None: +def _handle_exceptions( + e: Exception, error_msg: str = "", client: Optional[httpx.Client] = None +) -> None: conn = object.__new__(_ConnectionBase) - # keep the bare instance's __del__ quiet (it checks these for unclosed connections) - conn._client = None + # __del__ checks these for unclosed connections; None also means "no client at all" + conn._client = client conn._grpc_channel = None getattr(conn, "_ConnectionBase__handle_exceptions")(e, error_msg) # noqa: B009 -def test_httpx_closed_client_runtime_error_maps_to_closed_client() -> None: - # the exact message httpx raises for a closed AsyncClient/Client +def test_runtime_error_from_a_closed_client_maps_to_closed_client() -> None: + # httpx raises a bare RuntimeError('Cannot send a request, as the client has been + # closed.'); the client's state, not the message text, is what makes it 'closed' + closed = httpx.Client() + closed.close() with pytest.raises(WeaviateClosedClientError): _handle_exceptions(RuntimeError("Cannot send a request, as the client has been closed.")) + with pytest.raises(WeaviateClosedClientError): + _handle_exceptions(RuntimeError("some other wording"), client=closed) def test_unrelated_runtime_error_is_not_rewritten_as_closed_client() -> None: # Emscripten's canonical thread failure must propagate as-is, not as a misleading # 'client is closed - run client.connect()' - with pytest.raises(RuntimeError, match="can't start new thread"): - _handle_exceptions(RuntimeError("can't start new thread")) + with httpx.Client() as open_client: + with pytest.raises(RuntimeError, match="can't start new thread"): + _handle_exceptions(RuntimeError("can't start new thread"), client=open_client) def test_connect_error_message_includes_exception_type() -> None: @@ -206,21 +216,52 @@ def test_rest_timeouts_are_capped_at_five_seconds_on_native_platforms() -> None: assert insert.read == 90 -def test_rest_connect_timeout_follows_the_request_timeout_under_emscripten(monkeypatch) -> None: - # under Pyodide the connect timeout bounds the entire fetch promise, so a fixed 5s - # silently caps every request at ~5s of wall clock +def test_rest_deadline_under_emscripten_is_the_read_timeout(monkeypatch) -> None: + # the fetch transport (weaviate-client-web) reads only `read` as the whole-request + # deadline; connect/write/pool are never consulted there, so nothing platform-specific + # is computed: same httpx.Timeout as on CPython, finite -> value, non-finite -> None monkeypatch.setattr(sys, "platform", "emscripten") - conn = _connection() - insert = _get_timeout(conn, "POST") - assert insert.connect == 90 and insert.write == 90 - assert insert.read == 90 - - query = _get_timeout(conn, "GET") - assert query.connect == 30 + insert = _get_timeout(_connection(), "POST") + assert insert.read == 90 and insert.connect == 5.0 + query = _get_timeout(_connection(), "GET") assert query.read == 30 - # a configured timeout below the httpx default must not make connecting stricter short = _get_timeout(_connection(insert=1, query=1), "POST") - assert short.connect == 5.0 assert short.read == 1 + unbounded = _get_timeout(_connection(insert=float("inf")), "POST") + assert unbounded.read is None + + +@pytest.mark.parametrize("value", [float("inf"), float("nan")]) +def test_deadline_maps_non_finite_to_none(value: float) -> None: + # Timeout(query=float('inf')) passes pydantic's ge=0; handed on as-is it overflows in + # the fetch layer under Pyodide (int(inf * 1000)). None is "no deadline" for both + # httpx and grpc. + assert _deadline(value) is None + assert _deadline(None) is None + assert _deadline(0) == 0 + assert _deadline(2.5) == 2.5 + + +def test_infinite_rest_timeouts_become_no_read_timeout() -> None: + conn = _connection(insert=float("inf"), query=float("inf")) + insert = _get_timeout(conn, "POST") + assert insert.read is None + assert insert.connect == 5.0 # the httpx default is kept on native platforms + query = _get_timeout(conn, "GET") + assert query.read is None + + +def test_deadlines_view_sanitises_every_timeout() -> None: + conn = _connection() + conn.timeout_config = TimeoutConfig( + query=float("inf"), insert=90, init=float("inf"), stream=float("inf") + ) + deadlines = conn._deadlines + assert deadlines.query is None + assert deadlines.insert == 90 + assert deadlines.init is None # Timeout(init=inf) used to end in an OverflowError + assert deadlines.stream is None + # the user's own config object is left untouched + assert conn.timeout_config.init == float("inf") diff --git a/weaviate/connect/base.py b/weaviate/connect/base.py index fd5bed8e5..4fc093023 100644 --- a/weaviate/connect/base.py +++ b/weaviate/connect/base.py @@ -149,6 +149,29 @@ def _grpc_web_path_prefix(self) -> str: cleaned = (self.grpc_path_prefix or "").strip("/") return f"/{cleaned}" if cleaned else "" + def _check_grpc_web_usable(self, is_async: bool) -> None: + """Fail fast on a grpc-web prefix this process cannot honour; a no-op for native gRPC. + + A native grpcio channel would silently ignore the ``grpc-web.path_prefix`` option + and route over native gRPC, so the shim (which consumes it) must be in place. + """ + if self._grpc_web_path_prefix == "": + return + if not is_async: + raise WeaviateInvalidInputError( + "grpc_path_prefix (grpc-web) is only supported for async clients; " + "use use_async_with_custom(...) / WeaviateAsyncClient" + ) + if not _grpc_web_shim_active(): + raise WeaviateInvalidInputError( + "grpc_path_prefix enables grpc-web, which requires the " + "'weaviate-client-web' package (it installs a grpc shim before " + "'import weaviate'); it is not active in this environment. Under Pyodide a " + "plain `import weaviate` activates it; on CPython call " + "weaviate_client_web.install(force=True) and set_sender(make_httpx_sender()) " + "before importing weaviate (intended for integration testing)." + ) + def _grpc_channel( self, proxies: Dict[str, str], @@ -172,24 +195,9 @@ def _grpc_channel( if grpc_config is not None and grpc_config.channel_options is not None: options.extend(grpc_config.channel_options) - # grpc-web mode (prefix set): only valid for an async client, and only when the - # weaviate-client-web shim has replaced the grpc module (it consumes the - # grpc-web.path_prefix option). Fail fast otherwise — a native grpcio channel - # would silently ignore the option and route over native gRPC, a confusing - # misconfiguration. Nothing is added for native gRPC, so its channel options stay - # byte-for-byte unchanged. + # nothing is added for native gRPC, so its channel options stay byte-for-byte unchanged if (prefix := self._grpc_web_path_prefix) != "": - if not is_async: - raise WeaviateInvalidInputError( - "grpc_path_prefix (grpc-web) is only supported for async clients; " - "use use_async_with_custom(...) / WeaviateAsyncClient" - ) - if not _grpc_web_shim_active(): - raise WeaviateInvalidInputError( - "grpc_path_prefix enables grpc-web, which requires the " - "'weaviate-client-web' package (it installs a grpc shim before " - "'import weaviate'); it is not active in this environment" - ) + self._check_grpc_web_usable(is_async) options.append(("grpc-web.path_prefix", prefix)) if is_async: diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index a2d87afdb..28153ddc1 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -351,22 +351,23 @@ def connect_to_custom( True >>> # The connection is automatically closed when the context is exited. """ - if grpc_path_prefix: + connection_params = ConnectionParams.from_params( + http_host=http_host, + http_port=http_port, + http_secure=http_secure, + grpc_host=grpc_host, + grpc_port=grpc_port, + grpc_secure=grpc_secure, + grpc_path_prefix=grpc_path_prefix, + ) + if connection_params._grpc_web_path_prefix: # normalized: "/" is native gRPC raise WeaviateInvalidInputError( "grpc_path_prefix enables grpc-web, which is async-only; use " "use_async_with_custom(...) instead of connect_to_custom(...)" ) return __connect( WeaviateClient( - ConnectionParams.from_params( - http_host=http_host, - http_port=http_port, - http_secure=http_secure, - grpc_host=grpc_host, - grpc_port=grpc_port, - grpc_secure=grpc_secure, - grpc_path_prefix=grpc_path_prefix, - ), + connection_params, auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, additional_config=additional_config, diff --git a/weaviate/connect/v4.py b/weaviate/connect/v4.py index 852da8dd9..e0c4a7fca 100644 --- a/weaviate/connect/v4.py +++ b/weaviate/connect/v4.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import math import sys import time from copy import copy @@ -111,6 +112,44 @@ PERMISSION_DENIED = "PERMISSION_DENIED" +# ceiling (seconds) for the exponential backoff between failed token refresh attempts +TOKEN_REFRESH_BACKOFF_CAP = 60 + + +def _token_refresh_backoff(consecutive_failures: int) -> int: + """Seconds to wait before the next refresh attempt: 1, 2, 4, ... up to the cap.""" + return min(2 ** min(consecutive_failures - 1, 16), TOKEN_REFRESH_BACKOFF_CAP) + + +def _deadline(timeout: Union[int, float, None]) -> Union[int, float, None]: + """A timeout as handed to httpx/grpc: non-finite (e.g. ``float('inf')``) means no deadline. + + Passed on as-is, ``inf`` overflows in the fetch layer under Pyodide; None is "no + timeout" for httpx and "no deadline" for grpc alike. + """ + if timeout is None or not math.isfinite(timeout): + return None + return timeout + + +@dataclass(frozen=True) +class _Deadlines: + """The user's timeout config as handed to httpx/grpc (each value through ``_deadline``).""" + + query: Union[int, float, None] + insert: Union[int, float, None] + init: Union[int, float, None] + stream: Union[int, float, None] + + @classmethod + def of(cls, config: TimeoutConfig) -> "_Deadlines": + return cls( + query=_deadline(config.query), + insert=_deadline(config.insert), + init=_deadline(config.init), + stream=_deadline(config.stream), + ) + def _exc_detail(e: BaseException) -> str: """Format an exception for user-facing messages. @@ -158,6 +197,8 @@ def __init__( self._connection_params = connection_params self._grpc_stub: Optional[weaviate_pb2_grpc.WeaviateStub] = None self._grpc_channel: Union[AsyncChannel, SyncChannel, None] = None + # a grpc-web prefix this process cannot honour fails here, not deep inside connect() + connection_params._check_grpc_web_usable(is_async=not isinstance(self, ConnectionSync)) self.timeout_config = timeout_config self.__connection_config = connection_config self.__trust_env = trust_env @@ -328,7 +369,7 @@ def _ping_grpc(self, colour: executor.Colour) -> Union[None, Awaitable[None]]: "/grpc.health.v1.Health/Check", request_serializer=health_weaviate_pb2.WeaviateHealthCheckRequest.SerializeToString, response_deserializer=health_weaviate_pb2.WeaviateHealthCheckResponse.FromString, - )(health_weaviate_pb2.WeaviateHealthCheckRequest(), timeout=self.timeout_config.init) + )(health_weaviate_pb2.WeaviateHealthCheckRequest(), timeout=self._deadlines.init) if colour == "async": async def execute(): @@ -388,6 +429,10 @@ def server_version(self) -> str: """Version of the weaviate instance.""" return str(self._weaviate_version) + @property + def _deadlines(self) -> _Deadlines: + return _Deadlines.of(self.timeout_config) + def get_proxies(self) -> Dict[str, str]: return self._proxies @@ -432,7 +477,7 @@ def _open_connections_rest( async def get_oidc() -> None: async with self._make_client("async") as client: try: - response = await client.get(oidc_url, timeout=self.timeout_config.init) + response = await client.get(oidc_url, timeout=self._deadlines.init) except Exception as e: raise WeaviateConnectionError( f"Error: {_exc_detail(e)}. \nIs Weaviate running and reachable at {self.url}?" @@ -447,7 +492,7 @@ async def get_oidc() -> None: with self._make_client("sync") as client: try: - response = client.get(oidc_url, timeout=self.timeout_config.init) + response = client.get(oidc_url, timeout=self._deadlines.init) except Exception as e: raise WeaviateConnectionError( f"Error: {_exc_detail(e)}. \nIs Weaviate running and reachable at {self.url}?" @@ -557,15 +602,17 @@ def _create_background_token_refresh(self, _auth: Optional[_Auth] = None) -> Non if "refresh_token" not in self._client.token and _auth is None: return - # a previous connect() may have left a refresher behind (e.g. a retry after a - # partially failed connect); stop it before replacing the shutdown event, or it - # would keep refreshing concurrently forever + # stop the refresher a previous connect() may have left behind (e.g. a retry + # after a partially failed connect) self._cancel_background_token_refresh() expires_in: int = self._client.token.get( "expires_in", 60 ) # use 1minute as token lifetime if not supplied - self._shutdown_background_event = Event() + # captured by the refresher below, so that a later close()+connect() (which + # replaces the attribute) still stops THIS refresher and not the new one + shutdown = Event() + self._shutdown_background_event = shutdown if isinstance(self._client, AsyncOAuth2Client): try: @@ -575,7 +622,9 @@ def _create_background_token_refresh(self, _auth: Optional[_Auth] = None) -> Non if loop is not None: # The async colour refreshes on its own already-running loop: threads # cannot start under WASM/Pyodide and are unnecessary here anyway. - task = loop.create_task(self.__periodic_token_refresh_async(expires_in, _auth)) + task = loop.create_task( + self.__periodic_token_refresh_async(expires_in, _auth, shutdown) + ) task.add_done_callback(self.__warn_if_token_refresh_died) self.__token_refresh_task = task return @@ -617,12 +666,9 @@ def update_refresh_time() -> int: return self._client.token.get("expires_in", 60) - 30 def periodic_refresh_token(refresh_time: int, _auth: Optional[_Auth]) -> None: - while ( - self._shutdown_background_event is not None - and not self._shutdown_background_event.is_set() - ): - # use refresh token when available - time.sleep(max(refresh_time, 1)) + failures = 0 + # Event.wait instead of time.sleep so close() ends the thread promptly + while not shutdown.wait(timeout=max(refresh_time, 1)): try: if self._client is None: continue @@ -636,10 +682,12 @@ def periodic_refresh_token(refresh_time: int, _auth: Optional[_Auth]) -> None: # saved credentials refresh_session() refresh_time = update_refresh_time() - except HTTPError as exc: - # retry again after one second, might be an unstable connection - refresh_time = 1 - _Warnings.token_refresh_failed(exc) + failures = 0 + except Exception as exc: + # any failure (not only transport errors) keeps the refresher alive + failures += 1 + refresh_time = _token_refresh_backoff(failures) + _Warnings.token_refresh_failed(exc, refresh_time, failures) demon = Thread( target=periodic_refresh_token, @@ -649,25 +697,38 @@ def periodic_refresh_token(refresh_time: int, _auth: Optional[_Auth]) -> None: ) demon.start() - def _cancel_background_token_refresh(self) -> None: + def _cancel_background_token_refresh(self) -> Optional["asyncio.Task[None]"]: """Stop the token refresher, whichever form it took. - Sets the shutdown event (observed by the sync colour's daemon thread at its next - wake-up) and cancels the async colour's refresh task immediately. + Sets the shutdown event (ends the sync colour's daemon thread) and cancels the + async colour's refresh task. Returns that task when it belongs to the running + loop so the caller can await its wind-down; a task on another loop is only asked + to cancel (thread-safely) and never awaited, which would hang. """ if self._shutdown_background_event is not None: self._shutdown_background_event.set() - if self.__token_refresh_task is not None: - self.__token_refresh_task.cancel() - self.__token_refresh_task = None + task, self.__token_refresh_task = self.__token_refresh_task, None + if task is None: + return None + try: + running: Optional[asyncio.AbstractEventLoop] = asyncio.get_running_loop() + except RuntimeError: + running = None + loop = task.get_loop() + if loop is running: + task.cancel() + return task + if not loop.is_closed(): + loop.call_soon_threadsafe(task.cancel) + return None @staticmethod def __warn_if_token_refresh_died(task: "asyncio.Task[None]") -> None: """Surface a refresher death that happened outside the loop body's own handler. - Nothing ever awaits this task and ``_cancel_background_token_refresh`` drops the - reference, so without this callback not even asyncio's "exception was never - retrieved" message fires — the client just silently stops refreshing. + Nothing awaits this task while it runs (close() only gathers it with + return_exceptions=True), so without this callback not even asyncio's "exception + was never retrieved" message fires — the client just silently stops refreshing. """ if task.cancelled(): return @@ -676,17 +737,14 @@ def __warn_if_token_refresh_died(task: "asyncio.Task[None]") -> None: _Warnings.token_refresh_stopped(exc) async def __periodic_token_refresh_async( - self, refresh_time: int, _auth: Optional[_Auth] + self, refresh_time: int, _auth: Optional[_Auth], shutdown: Event ) -> None: """Thread-free equivalent of ``periodic_refresh_token`` for the async colour. Cancelled by ``close('async')``. """ - while ( - self._shutdown_background_event is not None - and not self._shutdown_background_event.is_set() - ): - # use refresh token when available + failures = 0 + while not shutdown.is_set(): await asyncio.sleep(max(refresh_time, 1)) try: client = self._client @@ -701,17 +759,16 @@ async def __periodic_token_refresh_async( new_session = await _Auth.aresult(_auth.get_auth_session()) client.token = await new_session.fetch_token() refresh_time = client.token.get("expires_in", 60) - 30 + failures = 0 except asyncio.CancelledError: # close() cancels this task — cancellation must never be swallowed raise except Exception as exc: - # retry again after one second, might be an unstable connection. - # Catch broadly: an authlib OAuthError (e.g. invalid_grant), a KeyError on - # a missing token_endpoint or a malformed IdP body would otherwise kill the - # task with nobody watching, and every later request would 401 with nothing - # pointing at the token refresh. - refresh_time = 1 - _Warnings.token_refresh_failed(exc) + # any failure (an authlib OAuthError such as invalid_grant, a KeyError on a + # malformed IdP body, ...) keeps the refresher alive + failures += 1 + refresh_time = _token_refresh_backoff(failures) + _Warnings.token_refresh_failed(exc, refresh_time, failures) def __get_latest_headers(self) -> Dict[str, str]: if "authorization" in self._headers: @@ -740,42 +797,32 @@ def __get_timeout( They specify the times depending on how they expect Weaviate to behave. For example, a query might take longer than an insert or vice versa but, in either case, the user only cares about how long it takes for a response to be received. - The one exception is Emscripten/Pyodide, where the connect timeout bounds the whole fetch - promise rather than connection setup, so it has to follow the request timeout (see below). + Under Emscripten/Pyodide the fetch transport (weaviate-client-web) reads only `read`, + so it is the whole-request deadline there; a non-finite value means no deadline. https://www.python-httpx.org/advanced/timeouts/ """ + deadlines = self._deadlines timeout = None if method == "DELETE" or method == "PATCH" or method == "PUT": - timeout = self.timeout_config.insert + timeout = deadlines.insert elif method == "GET" or method == "HEAD": - timeout = self.timeout_config.query + timeout = deadlines.query elif method == "POST" and is_gql_query: - timeout = self.timeout_config.query + timeout = deadlines.query elif method == "POST" and not is_gql_query: - timeout = self.timeout_config.insert - - connect: float = 5.0 - if sys.platform == "emscripten" and timeout is not None: - # Under Pyodide the REST transport is httpx's AsyncJavascriptFetchTransport, - # where the connect timeout bounds the WHOLE fetch promise (upload + response) - # rather than connection setup only. A fixed 5s there silently caps every - # request at ~5s of wall clock and ignores the configured insert/query - # timeouts. Never below 5s, and untouched off Emscripten, so native platforms - # keep the httpx default for connect/write. - connect = max(connect, float(timeout)) + timeout = deadlines.insert + return Timeout( - timeout=connect, + timeout=5.0, read=timeout, pool=self.__connection_config.session_pool_timeout, ) def __handle_exceptions(self, e: Exception, error_msg: str) -> None: - # httpx raises a bare RuntimeError('Cannot send a request, as the client has been - # closed.'); match its message so unrelated RuntimeErrors (e.g. Emscripten's - # "can't start new thread") are not rewritten into a misleading 'client is - # closed' error. - if isinstance(e, RuntimeError) and "client has been closed" in str(e): + # httpx raises a bare RuntimeError for a closed client; only rewrite when the + # client really is closed, so unrelated RuntimeErrors keep their meaning + if isinstance(e, RuntimeError) and (self._client is None or self._client.is_closed): raise WeaviateClosedClientError() from e if isinstance(e, ConnectError): raise WeaviateConnectionError(self.__error_msg_with_detail(error_msg, e)) from e @@ -848,10 +895,14 @@ def exc(e: Exception) -> None: def close(self, colour: executor.Colour) -> executor.Result[None]: if self.embedded_db is not None: self.embedded_db.stop() - self._cancel_background_token_refresh() + refresh_task = self._cancel_background_token_refresh() if colour == "async": async def execute() -> None: + if refresh_task is not None: + # let the cancellation land before the client goes away; the task's + # own outcome (cancelled, or dead earlier) is not close()'s error + await asyncio.gather(refresh_task, return_exceptions=True) if self._client is not None: assert isinstance(self._client, AsyncClient) await self._client.aclose() @@ -889,7 +940,7 @@ def resp(res: Response) -> None: async def _execute() -> None: try: async with AsyncClient() as client: - res = await client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) + res = await client.get(PYPI_PACKAGE_URL, timeout=self._deadlines.init) return resp(res) except (RequestError, OSError): # ignore any errors related to requests, it is a best-effort warning. @@ -901,7 +952,7 @@ async def _execute() -> None: try: with Client() as client: - res = client.get(PYPI_PACKAGE_URL, timeout=self.timeout_config.init) + res = client.get(PYPI_PACKAGE_URL, timeout=self._deadlines.init) return resp(res) except (RequestError, OSError): pass # ignore any errors related to requests, it is a best-effort warning @@ -1039,13 +1090,22 @@ def resp(res: Response) -> Optional[Dict[str, Any]]: class ConnectionSync(_ConnectionBase): """Connection class used to communicate to a weaviate instance.""" - def __init__(self, *args: Any, **kwargs: Any) -> None: + def __init__( + self, + connection_params: ConnectionParams, + auth_client_secret: Optional[AuthCredentials], + timeout_config: TimeoutConfig, + proxies: Union[str, Proxies, None], + trust_env: bool, + additional_headers: Optional[Dict[str, Any]], + connection_config: ConnectionConfig, + embedded_db: Optional[EmbeddedV4] = None, + skip_init_checks: bool = False, + grpc_config: Optional[GrpcConfig] = None, + ) -> None: if sys.platform == "emscripten": - # Fail at construction with the async-only message; otherwise the first - # REST call surfaces an opaque ConnectError long before the grpc-web - # shim's own sync guard is reached (wording mirrors the shim's - # _ASYNC_ONLY_MESSAGE). Pre-set the attributes __del__ reads so the - # never-initialized instance is collected quietly. + # fail at construction, before the first REST call surfaces an opaque + # ConnectError; pre-set what __del__ reads so the instance is collected quietly self._client = None self._grpc_channel = None raise WeaviateStartUpError( @@ -1054,7 +1114,18 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: "use_async_with_weaviate_cloud / use_async_with_custom, or " "WeaviateAsyncClient) instead." ) - super().__init__(*args, **kwargs) + super().__init__( + connection_params=connection_params, + auth_client_secret=auth_client_secret, + timeout_config=timeout_config, + proxies=proxies, + trust_env=trust_env, + additional_headers=additional_headers, + connection_config=connection_config, + embedded_db=embedded_db, + skip_init_checks=skip_init_checks, + grpc_config=grpc_config, + ) def connect(self, force: bool = False) -> None: if self._connected and not force: @@ -1137,7 +1208,7 @@ def grpc_search(self, request: search_get_pb2.SearchRequest) -> search_get_pb2.S self.grpc_stub.Search, request, metadata=self.grpc_headers(), - timeout=self.timeout_config.query, + timeout=self._deadlines.query, ) return cast(search_get_pb2.SearchReply, res) except RpcError as e: @@ -1162,7 +1233,7 @@ def grpc_batch_objects( f=self.grpc_stub.BatchObjects, request=request, metadata=self.grpc_headers(), - timeout=timeout, + timeout=_deadline(timeout), ) res = cast(batch_pb2.BatchObjectsReply, res) @@ -1184,7 +1255,7 @@ def grpc_batch_stream( assert self.grpc_stub is not None for msg in self.grpc_stub.BatchStream( request_iterator=requests, - timeout=self.timeout_config.stream, + timeout=self._deadlines.stream, metadata=self.grpc_headers(), ): yield msg @@ -1206,7 +1277,7 @@ def grpc_batch_delete( self.grpc_stub.BatchDelete( request, metadata=self.grpc_headers(), - timeout=self.timeout_config.insert, + timeout=self._deadlines.insert, ), ) except RpcError as e: @@ -1226,7 +1297,7 @@ def grpc_tenants_get( self.grpc_stub.TenantsGet, request, metadata=self.grpc_headers(), - timeout=self.timeout_config.query, + timeout=self._deadlines.query, ) except RpcError as e: error = cast(Call, e) @@ -1247,7 +1318,7 @@ def grpc_aggregate( self.grpc_stub.Aggregate, request, metadata=self.grpc_headers(), - timeout=self.timeout_config.query, + timeout=self._deadlines.query, ) return cast(aggregate_pb2.AggregateReply, res) except RpcError as e: @@ -1348,7 +1419,7 @@ async def grpc_search( self.grpc_stub.Search, request, metadata=self.grpc_headers(), - timeout=self.timeout_config.query, + timeout=self._deadlines.query, ) return cast(search_get_pb2.SearchReply, res) except AioRpcError as e: @@ -1372,7 +1443,7 @@ async def grpc_batch_objects( f=self.grpc_stub.BatchObjects, request=request, metadata=self.grpc_headers(), - timeout=timeout, + timeout=_deadline(timeout), ) res = cast(batch_pb2.BatchObjectsReply, res) @@ -1383,7 +1454,7 @@ async def grpc_batch_objects( except AioRpcError as e: if e.code().name == PERMISSION_DENIED: raise InsufficientPermissionsError(e) - raise WeaviateBatchError(str(e)) from e + raise WeaviateBatchError(str(e.details())) from e async def grpc_batch_delete( self, request: batch_delete_pb2.BatchDeleteRequest @@ -1393,7 +1464,7 @@ async def grpc_batch_delete( return await self.grpc_stub.BatchDelete( request, metadata=self.grpc_headers(), - timeout=self.timeout_config.insert, + timeout=self._deadlines.insert, ) except AioRpcError as e: if e.code().name == PERMISSION_DENIED: @@ -1412,7 +1483,7 @@ async def grpc_batch_stream( response_deserializer=batch_pb2.BatchStreamReply.FromString, )( request_iterator=requests, - timeout=self.timeout_config.stream, + timeout=self._deadlines.stream, metadata=self.grpc_headers(), ): yield msg @@ -1467,7 +1538,7 @@ async def grpc_tenants_get( self.grpc_stub.TenantsGet, request, metadata=self.grpc_headers(), - timeout=self.timeout_config.query, + timeout=self._deadlines.query, ) except AioRpcError as e: if e.code().name == PERMISSION_DENIED: @@ -1487,7 +1558,7 @@ async def grpc_aggregate( self.grpc_stub.Aggregate, request, metadata=self.grpc_headers(), - timeout=self.timeout_config.query, + timeout=self._deadlines.query, ) return cast(aggregate_pb2.AggregateReply, res) except AioRpcError as e: diff --git a/weaviate/warnings.py b/weaviate/warnings.py index 053bac634..a1dd3180b 100644 --- a/weaviate/warnings.py +++ b/weaviate/warnings.py @@ -67,13 +67,11 @@ def auth_cannot_parse_oidc_config(url: str) -> None: warnings.warn(message=msg, category=UserWarning, stacklevel=1) @staticmethod - def token_refresh_failed(exc: Exception) -> None: + def token_refresh_failed(exc: Exception, retry_in: float, failures: int) -> None: + detail = f"{type(exc).__name__}: {exc}" if str(exc) else repr(exc) warnings.warn( - message=f"""Con001: Could not reach token issuer for the periodic refresh. This client will automatically - retry to refresh. If the retry does not succeed, the client will become unauthenticated. - - The cause might be an unstable internet connection or a problem with your authentication provider. - Exception: {exc} + message=f"""Con001: Token refresh failed ({detail}); retrying in {retry_in}s (consecutive failures: {failures}). + The client will become unauthenticated once the current token expires if the refresh keeps failing. """, category=UserWarning, stacklevel=1, From c099eb3cdd738e4b3d81bfed7f81d1c9255485d2 Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:35:17 +0200 Subject: [PATCH 25/27] fix(batch): keep the caller's exception on stream exit; sync/async parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leaving `with`/`async with client.batch.stream()` no longer replaces the exception raised in the block (CancelledError included) with the background failure — the caller's wins and the background one is logged, unless it is the same exception already propagating. Both colours raise WeaviateBatchStreamError when the workers failed or the stream ended with data still queued, with a message that says how much was unsent; the sync colour previously swallowed this. _BatchStreamShutdownError joins the taxonomy, the shutdown wait tolerates an infinite insert timeout, and WeaviateBatchError carries the gRPC details only. --- test/test_batch_async.py | 179 +++++++++++++++++++- test/test_batch_sync.py | 164 ++++++++++++++++++ weaviate/collections/batch/async_.py | 40 ++--- weaviate/collections/batch/batch_wrapper.py | 29 +++- weaviate/collections/batch/client.py | 10 ++ weaviate/collections/batch/collection.py | 10 ++ weaviate/collections/batch/sync.py | 21 ++- weaviate/exceptions.py | 7 +- 8 files changed, 432 insertions(+), 28 deletions(-) create mode 100644 test/test_batch_sync.py diff --git a/test/test_batch_async.py b/test/test_batch_async.py index 5ad611b94..cc8a9a3f2 100644 --- a/test/test_batch_async.py +++ b/test/test_batch_async.py @@ -7,13 +7,16 @@ """ import asyncio +import logging +from typing import Optional import grpc import pytest from weaviate.collections.batch.async_ import _BatchBaseAsync, _BgTasks from weaviate.collections.batch.base import _BatchDataWrapper -from weaviate.exceptions import WeaviateBatchStreamError +from weaviate.collections.batch.batch_wrapper import _ContextManagerAsync +from weaviate.exceptions import WeaviateBatchStreamError, _BatchStreamShutdownError class _NotAnException(BaseException): @@ -155,3 +158,177 @@ class FakeConnection: with pytest.raises(RuntimeError, match="boom"): asyncio.run(batch._wait()) assert backup.failed_objects == ["sentinel-failure"] + + +async def _finished_task() -> "asyncio.Task[None]": + """A background task that returned normally (e.g. the server closed the stream).""" + task = asyncio.get_running_loop().create_task(asyncio.sleep(0)) + await task + return task + + +class _FakeTimeouts: + insert = 1 + + +class _FakeConnection: + timeout_config = _FakeTimeouts() + + +def test_wait_names_unsent_data_when_the_tasks_ended_cleanly() -> None: + # both tasks returned normally (server closed the stream) with data still queued: + # that is not "the background tasks died unexpectedly", it is an early end of the + # stream — say what was left behind + async def run() -> None: + batch = _bare_batch( + bg_exception=None, + bg_tasks=_BgTasks(recv=await _finished_task(), loop=await _finished_task()), + connection=_FakeConnection(), + results_for_wrapper=_BatchDataWrapper(), + results_for_wrapper_backup=_BatchDataWrapper(), + batch_objects=[object(), object()], + batch_references=[object()], + ) + await batch._wait() + + with pytest.raises( + WeaviateBatchStreamError, match="ended with 2 objects and 1 references unsent" + ): + asyncio.run(run()) + + +def test_check_alive_after_a_clean_end_says_the_stream_ended() -> None: + async def run() -> None: + batch = _bare_batch( + bg_exception=None, + bg_tasks=_BgTasks(recv=await _finished_task(), loop=await _finished_task()), + ) + getattr(batch, "_BatchBaseAsync__check_bg_tasks_alive")() # noqa: B009 + + with pytest.raises(WeaviateBatchStreamError, match="stream has ended"): + asyncio.run(run()) + + +def test_put_gives_up_when_the_tasks_are_dead() -> None: + # a full queue with a dead receiver never drains: __put must return False instead of + # retrying (and recursing) once per second forever + async def run() -> bool: + reqs: asyncio.Queue = asyncio.Queue(maxsize=1) + await reqs.put(object()) # full + batch = _bare_batch( + reqs=reqs, + bg_exception=None, # nothing recorded ... + shutdown_loop=asyncio.Event(), # ... and no shutdown either + bg_tasks=_BgTasks(recv=await _dead_task("cancel"), loop=await _finished_task()), + ) + put = getattr(batch, "_BatchBaseAsync__put") # noqa: B009 + return await asyncio.wait_for(put(object()), timeout=5) + + assert asyncio.run(run()) is False + + +def test_batch_stream_shutdown_error_is_in_the_taxonomy() -> None: + # raised on gRPC ABORTED and can surface from a clean `async with` exit + assert issubclass(_BatchStreamShutdownError, WeaviateBatchStreamError) + assert isinstance(_BatchStreamShutdownError(), Exception) + + +class _FakeBatch: + """Stands in for _BatchBaseAsync behind the context manager.""" + + def __init__(self, wait_error: Optional[BaseException] = None) -> None: + self.wait_error = wait_error + self.shutdown_called = False + self.wait_called = False + + async def _start(self) -> None: + pass + + async def _shutdown(self) -> None: + self.shutdown_called = True + + async def _wait(self) -> None: + self.wait_called = True + if self.wait_error is not None: + raise self.wait_error + + +def test_aexit_raises_a_background_failure_on_a_clean_block() -> None: + fake = _FakeBatch(WeaviateBatchStreamError("bg died")) + + async def run() -> None: + async with _ContextManagerAsync(fake): # type: ignore[arg-type] + pass + + with pytest.raises(WeaviateBatchStreamError, match="bg died"): + asyncio.run(run()) + + +def test_aexit_keeps_the_users_exception_over_a_background_failure(caplog) -> None: + # the block's own exception must not be REPLACED by the background failure (which + # used to demote it to __context__); the failure is logged instead + fake = _FakeBatch(WeaviateBatchStreamError("bg died")) + + async def run() -> None: + async with _ContextManagerAsync(fake): # type: ignore[arg-type] + raise ValueError("user code") + + with caplog.at_level(logging.WARNING, logger="weaviate-client"): + with pytest.raises(ValueError, match="user code"): + asyncio.run(run()) + assert fake.shutdown_called and fake.wait_called # still drained/awaited + assert "bg died" in caplog.text + + +def test_aexit_never_swallows_cancellation() -> None: + # a CancelledError leaving the block used to be replaced by the background failure, + # i.e. the cancellation was swallowed + fake = _FakeBatch(WeaviateBatchStreamError("bg died")) + + async def run() -> None: + async with _ContextManagerAsync(fake): # type: ignore[arg-type] + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + asyncio.run(run()) + assert fake.wait_called + + +def test_wait_with_an_infinite_insert_timeout_does_not_raise() -> None: + # Timeout(insert=inf) means "no deadline": the shutdown wait must not turn it into an + # error (the sync colour's Thread.join(inf) overflows; keep both colours consistent) + from weaviate.config import Timeout as TimeoutConfig + + class InfiniteInsert: + timeout_config = TimeoutConfig(insert=float("inf")) + + async def run() -> None: + batch = _bare_batch( + bg_exception=None, + bg_tasks=_BgTasks(recv=await _finished_task(), loop=await _finished_task()), + connection=InfiniteInsert(), + results_for_wrapper=_BatchDataWrapper(), + results_for_wrapper_backup=_BatchDataWrapper(), + batch_objects=[], + batch_references=[], + ) + await batch._wait() + + asyncio.run(run()) + + +def test_aexit_does_not_log_the_exception_that_is_already_propagating(caplog) -> None: + # flush()/add_object raised __bg_exception inside the block; _wait() re-raises the + # SAME object on exit — that is not a second failure to log + err = WeaviateBatchStreamError("bg died") + fake = _FakeBatch(err) + + async def run() -> None: + async with _ContextManagerAsync(fake): # type: ignore[arg-type] + raise err + + with caplog.at_level(logging.WARNING, logger="weaviate-client"): + with pytest.raises(WeaviateBatchStreamError, match="bg died"): + asyncio.run(run()) + assert fake.wait_called + assert "batch stream failed" not in caplog.text diff --git a/test/test_batch_sync.py b/test/test_batch_sync.py new file mode 100644 index 000000000..5656e7388 --- /dev/null +++ b/test/test_batch_sync.py @@ -0,0 +1,164 @@ +"""Unit tests for the sync batch-stream failure handling, mirroring test_batch_async.py. + +_wait() surfaces a background failure (or data left unsent) instead of returning as if +the batch had succeeded, while an exception leaving the `with` block still wins. +""" + +import logging +import threading +import time +from typing import Optional + +import pytest + +from weaviate.collections.batch.base import _BatchDataWrapper, _BgThreads +from weaviate.collections.batch.batch_wrapper import _ContextManagerSync +from weaviate.collections.batch.sync import _BatchBaseSync +from weaviate.config import Timeout as TimeoutConfig +from weaviate.exceptions import WeaviateBatchStreamError + + +def _bare_batch(**mangled) -> _BatchBaseSync: + batch = object.__new__(_BatchBaseSync) + for name, value in mangled.items(): + setattr(batch, f"_BatchBaseSync__{name}", value) + return batch + + +class _FakeThreads: + def __init__(self, alive: bool = False) -> None: + self.alive = alive + + def join(self, timeout=None) -> None: + return None + + def is_alive(self) -> bool: + return self.alive + + +class _FakeTimeouts: + insert = 1 + + +class _FakeConnection: + timeout_config = _FakeTimeouts() + + +def _batch_for_wait(**mangled) -> _BatchBaseSync: + defaults = { + "bg_exception": None, + "bg_threads": _FakeThreads(alive=False), + "connection": _FakeConnection(), + "results_for_wrapper": _BatchDataWrapper(), + "results_for_wrapper_backup": _BatchDataWrapper(), + "batch_objects": [], + "batch_references": [], + } + defaults.update(mangled) + return _bare_batch(**defaults) + + +def test_wait_returns_quietly_when_everything_was_sent() -> None: + _batch_for_wait()._wait() + + +def test_wait_raises_the_background_exception_and_keeps_partial_results() -> None: + # like the async colour: a background failure must not come back as a success + partial = _BatchDataWrapper() + partial.failed_objects = ["sentinel-failure"] # type: ignore[list-item] + backup = _BatchDataWrapper() + batch = _batch_for_wait( + bg_exception=RuntimeError("boom"), + results_for_wrapper=partial, + results_for_wrapper_backup=backup, + ) + + with pytest.raises(RuntimeError, match="boom"): + batch._wait() + assert backup.failed_objects == ["sentinel-failure"] + + +def test_wait_names_unsent_data_when_the_threads_are_gone() -> None: + batch = _batch_for_wait(batch_objects=[object(), object()], batch_references=[object()]) + with pytest.raises( + WeaviateBatchStreamError, match="ended with 2 objects and 1 references unsent" + ): + batch._wait() + + +def test_check_alive_raises_inside_the_taxonomy() -> None: + # used to be a bare Exception("Batch thread died unexpectedly") + batch = _bare_batch(bg_exception=None, bg_threads=_FakeThreads(alive=False)) + with pytest.raises(WeaviateBatchStreamError, match="stream has ended"): + getattr(batch, "_BatchBaseSync__check_bg_threads_alive")() # noqa: B009 + + +class _FakeBatch: + def __init__(self, wait_error: Optional[BaseException] = None) -> None: + self.wait_error = wait_error + self.shutdown_called = False + self.wait_called = False + + def _start(self) -> None: + pass + + def _shutdown(self) -> None: + self.shutdown_called = True + + def _wait(self) -> None: + self.wait_called = True + if self.wait_error is not None: + raise self.wait_error + + +def test_exit_raises_a_background_failure_on_a_clean_block() -> None: + fake = _FakeBatch(WeaviateBatchStreamError("bg died")) + with pytest.raises(WeaviateBatchStreamError, match="bg died"): + with _ContextManagerSync(fake): # type: ignore[type-var] + pass + + +def test_exit_keeps_the_users_exception_over_a_background_failure(caplog) -> None: + fake = _FakeBatch(WeaviateBatchStreamError("bg died")) + with caplog.at_level(logging.WARNING, logger="weaviate-client"): + with pytest.raises(ValueError, match="user code"): + with _ContextManagerSync(fake): # type: ignore[type-var] + raise ValueError("user code") + assert fake.shutdown_called and fake.wait_called + assert "bg died" in caplog.text + + +def test_exit_never_swallows_a_base_exception() -> None: + fake = _FakeBatch(WeaviateBatchStreamError("bg died")) + with pytest.raises(KeyboardInterrupt): + with _ContextManagerSync(fake): # type: ignore[type-var] + raise KeyboardInterrupt() + assert fake.wait_called + + +def test_wait_with_an_infinite_insert_timeout_does_not_overflow_join() -> None: + # Timeout(insert=inf) is accepted (it means "no deadline"), but Thread.join(inf) + # raises OverflowError — the shutdown wait must become "no timeout" instead + class InfiniteInsert: + timeout_config = TimeoutConfig(insert=float("inf")) + + threads = _BgThreads( + loop=threading.Thread(target=lambda: None), recv=threading.Thread(target=lambda: None) + ) + threads.start_recv() + threads.start_loop() + time.sleep(0.1) # let both finish; is_alive()/join() are deliberately not called yet + _batch_for_wait(bg_threads=threads, connection=InfiniteInsert())._wait() + + +def test_exit_does_not_log_the_exception_that_is_already_propagating(caplog) -> None: + # flush()/add_object raised __bg_exception inside the block; _wait() re-raises the + # SAME object on exit — that is not a second failure to log + err = WeaviateBatchStreamError("bg died") + fake = _FakeBatch(err) + with caplog.at_level(logging.WARNING, logger="weaviate-client"): + with pytest.raises(WeaviateBatchStreamError, match="bg died"): + with _ContextManagerSync(fake): # type: ignore[type-var] + raise err + assert fake.wait_called + assert "batch stream failed" not in caplog.text diff --git a/weaviate/collections/batch/async_.py b/weaviate/collections/batch/async_.py index 21310ab94..d09bbf723 100644 --- a/weaviate/collections/batch/async_.py +++ b/weaviate/collections/batch/async_.py @@ -39,7 +39,7 @@ from weaviate.collections.classes.types import WeaviateProperties from weaviate.connect.base import _grpc_web_shim_active from weaviate.connect.executor import aresult -from weaviate.connect.v4 import ConnectionAsync +from weaviate.connect.v4 import ConnectionAsync, _deadline from weaviate.exceptions import ( WeaviateBatchFailedToReestablishStreamError, WeaviateBatchStreamError, @@ -194,7 +194,8 @@ async def recv_wrapper() -> None: async def _wait(self) -> None: assert self.__bg_tasks is not None # this is how long an insert will take to timeout for, so we wait at most this time +5s for the batch to finish after shutdown is initiated, in case the server never hangs up - shutdown_timeout = self.__connection.timeout_config.insert + 5 + insert = _deadline(self.__connection.timeout_config.insert) + shutdown_timeout = None if insert is None else insert + 5 try: await self.__bg_tasks.gather(timeout=shutdown_timeout) except asyncio.TimeoutError as e: @@ -217,13 +218,14 @@ async def _wait(self) -> None: # surface background-task failures instead of returning partial results # as if the batch had succeeded raise self.__bg_exception - if ( - len(self.__batch_objects) > 0 or len(self.__batch_references) > 0 - ) and not self.__all_tasks_alive(): - # the tasks are gone with data still queued and nothing recorded in - # __bg_exception (a BaseException escaping loop_wrapper/recv_wrapper): the - # batch did NOT complete, so do not return as if it had - raise self.__bg_task_death_cause() + n_objs, n_refs = len(self.__batch_objects), len(self.__batch_references) + if n_objs + n_refs > 0 and not self.__all_tasks_alive(): + # the tasks are gone with data still queued: the batch did NOT complete, so + # do not return as if it had — whether a task died (a BaseException escaping + # loop_wrapper/recv_wrapper) or the server ended the stream early + raise self.__bg_task_death_cause() or WeaviateBatchStreamError( + f"batch stream ended with {n_objs} objects and {n_refs} references unsent" + ) async def _shutdown(self) -> None: self.__is_stopped.set() @@ -560,12 +562,8 @@ async def flush(self) -> None: """Flush the batch queue and wait for all requests to be finished.""" # bg thread is sending objs+refs automatically, so simply wait for everything to be done while len(self.__batch_objects) > 0 or len(self.__batch_references) > 0: - # a dead task is the condition to check, not __bg_exception: loop_wrapper / - # recv_wrapper only catch Exception, so a BaseException (notably the - # asyncio.CancelledError grpc.aio raises on a cancelled streaming call) ends - # the task with __bg_exception unset. Nothing then drains the queues, so - # waiting any longer would hang forever. Mirrors the sync colour's - # __check_bg_threads_alive(). + # a dead task (not just __bg_exception, which a BaseException such as grpc.aio's + # CancelledError leaves unset) means nothing drains the queues: raise, don't hang self.__check_bg_tasks_alive() await asyncio.sleep(0.01) @@ -664,10 +662,14 @@ def __check_bg_tasks_alive(self) -> None: if self.__all_tasks_alive(): return - raise self.__bg_exception or self.__bg_task_death_cause() + raise ( + self.__bg_exception + or self.__bg_task_death_cause() + or WeaviateBatchStreamError("the batch stream has ended") + ) - def __bg_task_death_cause(self) -> Exception: - """Explain a background task that ended without setting __bg_exception. + def __bg_task_death_cause(self) -> Optional[Exception]: + """Explain a background task that died without setting __bg_exception; None if none did. loop_wrapper/recv_wrapper only catch Exception, so a BaseException — notably the asyncio.CancelledError grpc.aio raises when a streaming call is cancelled — ends @@ -689,4 +691,4 @@ def __bg_task_death_cause(self) -> Exception: return exc if exc is not None: return WeaviateBatchStreamError(f"the background {name} task died with {exc!r}") - return WeaviateBatchStreamError("the background tasks died unexpectedly") + return None diff --git a/weaviate/collections/batch/batch_wrapper.py b/weaviate/collections/batch/batch_wrapper.py index a3a3598d6..eb1353dd7 100644 --- a/weaviate/collections/batch/batch_wrapper.py +++ b/weaviate/collections/batch/batch_wrapper.py @@ -508,7 +508,18 @@ def __init__(self, current_batch: T): def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: self.__current_batch._shutdown() - self.__current_batch._wait() + if exc_type is None: + self.__current_batch._wait() + return + # the exception leaving the block wins; a background failure is only logged + # (unless it IS the one in flight, re-raised by _wait after flush/add already did) + try: + self.__current_batch._wait() + except Exception as e: + if e is not exc_val: + logger.warning( + f"batch stream failed while the block raised {exc_type.__name__}: {e}" + ) def __enter__(self) -> P: self.__current_batch._start() @@ -521,7 +532,21 @@ def __init__(self, current_batch: _BatchBaseAsync): async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: await self.__current_batch._shutdown() - await self.__current_batch._wait() + if exc_type is None: + await self.__current_batch._wait() + return + # the exception leaving the block wins (never replace a CancelledError or other + # BaseException); a background failure is only logged (unless it IS the one in + # flight, re-raised by _wait after flush/add already did) + try: + await self.__current_batch._wait() + except asyncio.CancelledError: + raise + except Exception as e: + if e is not exc_val: + logger.warning( + f"batch stream failed while the block raised {exc_type.__name__}: {e}" + ) async def __aenter__(self) -> Q: await self.__current_batch._start() diff --git a/weaviate/collections/batch/client.py b/weaviate/collections/batch/client.py index d9a2524d7..b95238e03 100644 --- a/weaviate/collections/batch/client.py +++ b/weaviate/collections/batch/client.py @@ -285,6 +285,11 @@ def stream( Args: concurrency: The number of concurrent streams to use when sending batches. If not provided, the default will be one. consistency_level: The consistency level to be used when inserting data. If not provided, the default value is `None`. + + Raises: + WeaviateBatchStreamError: On exit, if the background stream failed or ended with objects or references + still unsent. An exception raised inside the block is propagated unchanged; a background failure is + then only logged. """ if self._connection._weaviate_version.is_lower_than(1, 36, 0): raise WeaviateUnsupportedFeatureError( @@ -344,6 +349,11 @@ def stream( Args: concurrency: The number of concurrent streams to use when sending batches. If not provided, the default will be one. consistency_level: The consistency level to be used when inserting data. If not provided, the default value is `None`. + + Raises: + WeaviateBatchStreamError: On exit, if the background stream failed or ended with objects or references + still unsent. An exception raised inside the block is propagated unchanged; a background failure is + then only logged. """ if self._connection._weaviate_version.is_lower_than(1, 36, 0): raise WeaviateUnsupportedFeatureError( diff --git a/weaviate/collections/batch/collection.py b/weaviate/collections/batch/collection.py index 415b2df49..565412563 100644 --- a/weaviate/collections/batch/collection.py +++ b/weaviate/collections/batch/collection.py @@ -310,6 +310,11 @@ def stream( Args: concurrency: The number of concurrent requests when sending batches. This controls the number of concurrent requests made to Weaviate. If not provided, the default value is 1. + + Raises: + WeaviateBatchStreamError: On exit, if the background stream failed or ended with objects or references + still unsent. An exception raised inside the block is propagated unchanged; a background failure is + then only logged. """ if self._connection._weaviate_version.is_lower_than(1, 36, 0): raise WeaviateUnsupportedFeatureError( @@ -370,6 +375,11 @@ def stream( Args: concurrency: The number of concurrent requests when sending batches. This controls the number of concurrent requests made to Weaviate. If not provided, the default value is 1. + + Raises: + WeaviateBatchStreamError: On exit, if the background stream failed or ended with objects or references + still unsent. An exception raised inside the block is propagated unchanged; a background failure is + then only logged. """ if self._connection._weaviate_version.is_lower_than(1, 36, 0): raise WeaviateUnsupportedFeatureError( diff --git a/weaviate/collections/batch/sync.py b/weaviate/collections/batch/sync.py index 6cf8c1edc..40e53b6c0 100644 --- a/weaviate/collections/batch/sync.py +++ b/weaviate/collections/batch/sync.py @@ -35,7 +35,7 @@ ) from weaviate.collections.classes.types import WeaviateProperties from weaviate.connect.executor import result -from weaviate.connect.v4 import ConnectionSync +from weaviate.connect.v4 import ConnectionSync, _deadline from weaviate.exceptions import ( WeaviateBatchFailedToReestablishStreamError, WeaviateBatchStreamError, @@ -134,7 +134,8 @@ def _start(self) -> None: def _wait(self) -> None: # this is how long an insert will take to timeout for, so we wait at most this time +5s for the batch to finish after shutdown is initiated, in case the server never hangs up - shutdown_timeout = self.__connection.timeout_config.insert + 5 + insert = _deadline(self.__connection.timeout_config.insert) + shutdown_timeout = None if insert is None else insert + 5 # inf overflows Thread.join try: self.__bg_threads.join(shutdown_timeout) except TimeoutError as e: @@ -142,7 +143,8 @@ def _wait(self) -> None: "Background batch threads did not terminate after forced shutdown." ) from e - # copy the results to the public results + # copy the results to the public results — even on failure, so the user can + # still inspect batch.results / batch.failed_objects after catching self.__results_for_wrapper_backup.results = self.__results_for_wrapper.results self.__results_for_wrapper_backup.failed_objects = self.__results_for_wrapper.failed_objects self.__results_for_wrapper_backup.failed_references = ( @@ -152,6 +154,17 @@ def _wait(self) -> None: self.__results_for_wrapper.imported_shards ) + if self.__bg_exception is not None: + # surface background-thread failures instead of returning partial results + # as if the batch had succeeded + raise self.__bg_exception + n_objs, n_refs = len(self.__batch_objects), len(self.__batch_references) + if n_objs + n_refs > 0 and not self.__all_threads_alive(): + # the threads are gone with data still queued: the batch did NOT complete + raise WeaviateBatchStreamError( + f"batch stream ended with {n_objs} objects and {n_refs} references unsent" + ) + def _shutdown(self) -> None: # Shutdown the current batch and wait for all requests to be finished self.__is_stopped.set() @@ -644,4 +657,4 @@ def __check_bg_threads_alive(self) -> None: if self.__all_threads_alive(): return - raise self.__bg_exception or Exception("Batch thread died unexpectedly") + raise self.__bg_exception or WeaviateBatchStreamError("the batch stream has ended") diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index 2024a1258..3f0873a32 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -473,5 +473,8 @@ def __init__(self, pb: version.Version, grpc: version.Version) -> None: ) -class _BatchStreamShutdownError(Exception): - """Internal exception to signal that the batch stream was shutdown.""" +class _BatchStreamShutdownError(WeaviateBatchStreamError): + """Internal exception to signal that the batch stream was shutdown (gRPC ABORTED).""" + + def __init__(self, message: str = "the server aborted the batch stream") -> None: + super().__init__(message) From 8a18975e852100b7de732f4e5dd0a6227a638efd Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:35:32 +0200 Subject: [PATCH 26/27] ci: type-check and flake8 packages/web pyright now includes packages/web/src and the lint job's flake8 run covers the package alongside ruff, so the companion is held to the same bar as the client. --- .github/workflows/main.yaml | 2 +- pyrightconfig.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 240aaa417..163c70afa 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -49,7 +49,7 @@ jobs: - name: "Ruff format" run: ruff format --diff weaviate test mock_tests integration packages/web - name: "Flake 8" - run: flake8 weaviate test mock_tests integration + run: flake8 weaviate test mock_tests integration packages/web - name: "Check release for pypi" run: | python -m build diff --git a/pyrightconfig.json b/pyrightconfig.json index 396d62cfd..61eb3eaa3 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,6 +1,6 @@ { "include": [ - "weaviate", "integration" + "weaviate", "integration", "packages/web/src" ], "exclude": [ From 7b51604b7cb893cdc68aa1856f1fb995793996cd Mon Sep 17 00:00:00 2001 From: Ivan Despot <66276597+g-despot@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:33:43 +0200 Subject: [PATCH 27/27] feat(grpc-web): route gRPC over grpc-web automatically under WebAssembly Under Emscripten there is no grpcio wheel and no socket, so native gRPC cannot work at all and grpc-web on the REST listener is the only transport that can. The connect helpers now pin gRPC to the HTTP endpoint under Weaviate's own grpc-web base path, and `grpc_path_prefix` leaves the public API entirely. This matches the TypeScript web client, which hardcodes the same path in one place and removes grpcHost/grpcPort/grpcSecure from its connect options rather than exposing a knob. Weaviate serves grpc-web at a fixed path, so there was nothing for a caller to choose. use_async_with_local() and use_async_with_weaviate_cloud() now work in a browser as written; previously neither could reach grpc-web at all, and only the seven-argument use_async_with_custom() form could. Off Emscripten every helper is byte-identical to before, down to pydantic's echo of the constructor arguments in a port-collision validation error. Discarding a gRPC endpoint the caller actually chose warns (Con006) and names both endpoints; the helpers' own defaults are replaced silently, so the documented calls stay quiet. --- ci/pyodide-e2e/e2e.py | 11 +- packages/web/README.md | 76 +++++--- .../web/src/weaviate_client_web/__init__.py | 15 +- .../web/src/weaviate_client_web/_channel.py | 10 +- packages/web/tests/test_transport.py | 6 +- test/test_connection_params.py | 165 ++++++++++++++---- test/test_wasm_compat.py | 145 +++++++++++++++ weaviate/connect/helpers.py | 120 ++++++++----- weaviate/exceptions.py | 4 +- weaviate/warnings.py | 13 ++ 10 files changed, 445 insertions(+), 120 deletions(-) diff --git a/ci/pyodide-e2e/e2e.py b/ci/pyodide-e2e/e2e.py index ae92e40f5..b1f03ce9c 100644 --- a/ci/pyodide-e2e/e2e.py +++ b/ci/pyodide-e2e/e2e.py @@ -27,8 +27,9 @@ COLL = "PyodideE2E" MT_COLL = "PyodideE2ETenants" -# Weaviate core serves grpc-web natively on the REST port under this prefix -# (default-on since 1.38.3), so no proxy sits between the client and the server. +# Weaviate core serves grpc-web natively on the REST port under this prefix (default-on +# since 1.38.3), so no proxy sits between the client and the server. Under Emscripten the +# connect helpers route gRPC there themselves — nothing here selects it. GRPC_WEB_PREFIX = "/v1/grpc-web" @@ -58,8 +59,12 @@ async def main() -> None: grpc_host=host, grpc_port=port, grpc_secure=False, - grpc_path_prefix=GRPC_WEB_PREFIX, ) + params = client._connection._connection_params + assert params._grpc_web_path_prefix == GRPC_WEB_PREFIX, params + assert params._grpc_target == f"{host}:{port}", params + ok("connect helper routed gRPC onto the REST endpoint under /v1/grpc-web") + # No skip_init_checks: connect() performs the gRPC health check over grpc-web. await client.connect() ok("connect (health check over grpc-web)") diff --git a/packages/web/README.md b/packages/web/README.md index e1aa554f1..582ab2796 100644 --- a/packages/web/README.md +++ b/packages/web/README.md @@ -28,17 +28,33 @@ Under Pyodide there is no `grpcio` Emscripten wheel, and `import weaviate` hard- The `GrpcWebChannel` frames unary RPCs as grpc-web (a 5-byte header + protobuf payload) and POSTs them via `pyodide.http.pyfetch`. Call metadata (API key / OIDC bearer) is -folded into `fetch` headers. Two deployments work: - -1. **Core-native grpc-web (Weaviate ≥ 1.38.3).** Weaviate serves grpc-web on its REST - port under `/v1/grpc-web`, so gRPC and REST share one host, port and TLS setup and - no proxy is needed. Select it with `grpc_path_prefix="/v1/grpc-web"` and point - `grpc_host`/`grpc_port` at the REST endpoint. -2. **A grpc-web transcoder** (e.g. Envoy or - [connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)) listening at the - root of `grpc_host:grpc_port` in front of Weaviate's native gRPC port. This is what - `use_async_with_local()` / `use_async_with_weaviate_cloud()` need, because those - helpers have no `grpc_path_prefix` parameter. +folded into `fetch` headers. + +The target is not configurable: under Emscripten the connect helpers +(`use_async_with_local`, `use_async_with_weaviate_cloud`, `use_async_with_custom`) pin +gRPC to the **REST** endpoint — same host, port and TLS — under Weaviate's own +`/v1/grpc-web` base path, so gRPC and REST share one origin and no proxy is needed. That +is deliberate: native gRPC cannot work under WASM at all, so grpc-web on the REST +listener is not a choice that could be wrong. The TypeScript `@weaviate/web` client makes +the same call, dropping `grpcHost`/`grpcPort`/`grpcSecure` from its options entirely. + +A grpc-web transcoder on a separate endpoint (Envoy, +[connectrpc/vanguard](https://github.com/connectrpc/vanguard-go)) is therefore not +reachable through the helpers. If you need one — e.g. in front of a Weaviate older than +1.38.3 — build the connection parameters yourself: + +```python +from weaviate import WeaviateAsyncClient +from weaviate.connect import ConnectionParams + +client = WeaviateAsyncClient( + ConnectionParams.from_params( + http_host="weaviate.example.com", http_port=443, http_secure=True, + grpc_host="transcoder.example.com", grpc_port=443, grpc_secure=True, + # add grpc_path_prefix="/base/path" if the transcoder is not at the root + ) +) +``` For REST (`is_ready`, collection config, `/batch/references`, …) the package patches `httpx.AsyncHTTPTransport` with its own `pyfetch`-based transport. It does so even on @@ -57,24 +73,36 @@ is missing). Against Weaviate ≥ 1.38.3: ```python import weaviate # bootstraps weaviate_client_web automatically under Emscripten -client = weaviate.use_async_with_custom( - http_host="localhost", - http_port=8080, - http_secure=False, - grpc_host="localhost", # same host as REST - grpc_port=8080, # same port as REST: grpc-web rides on the REST listener - grpc_secure=False, - grpc_path_prefix="/v1/grpc-web", -) +client = weaviate.use_async_with_local(port=8080) await client.connect() # runs the gRPC health check over grpc-web collection = client.collections.get("Article") await collection.query.near_text("hello", limit=3) ``` -`use_async_with_local()` and `use_async_with_weaviate_cloud()` have no -`grpc_path_prefix`, so under Pyodide they only work when a grpc-web transcoder answers at -the root of `grpc_host:grpc_port`. Pass `headers={...}` / `auth_credentials=...` to -`use_async_with_custom` as usual for API keys, OIDC or WCD. +Nothing selects grpc-web: `use_async_with_local()`, `use_async_with_weaviate_cloud()` and +`use_async_with_custom()` all route gRPC onto the REST endpoint under `/v1/grpc-web` when +they run under Emscripten, and behave exactly as before everywhere else. + +```python +client = weaviate.use_async_with_weaviate_cloud( + cluster_url="rAnD0mD1g1t5.something.weaviate.cloud", + auth_credentials=weaviate.classes.init.Auth.api_key("my-api-key"), +) +``` + +`use_async_with_custom()` still requires `grpc_host`/`grpc_port`/`grpc_secure` — Python +cannot drop required parameters on one platform the way TypeScript drops them from a +type. Pass the HTTP values; anything else is overridden with them and warned about +(`Con006`), so a browser client never silently points somewhere it cannot reach. + +```python +client = weaviate.use_async_with_custom( + http_host="localhost", http_port=8080, http_secure=False, + grpc_host="localhost", grpc_port=8080, grpc_secure=False, # = the HTTP endpoint +) +``` + +Pass `headers={...}` / `auth_credentials=...` as usual for API keys, OIDC or WCD. Importing the companion explicitly first also works and remains the explicit form: diff --git a/packages/web/src/weaviate_client_web/__init__.py b/packages/web/src/weaviate_client_web/__init__.py index 24820d41d..2e9217991 100644 --- a/packages/web/src/weaviate_client_web/__init__.py +++ b/packages/web/src/weaviate_client_web/__init__.py @@ -12,16 +12,15 @@ import weaviate - client = weaviate.use_async_with_custom( - http_host="localhost", http_port=8080, http_secure=False, - grpc_host="localhost", grpc_port=8080, grpc_secure=False, - grpc_path_prefix="/v1/grpc-web", - ) + client = weaviate.use_async_with_local(port=8080) await client.connect() -``use_async_with_local`` / ``use_async_with_weaviate_cloud`` have no ``grpc_path_prefix`` -and therefore only work with a grpc-web transcoder (Envoy, vanguard) at the root of -``grpc_host:grpc_port``. +There is nothing to select. Under Emscripten ``use_async_with_local``, +``use_async_with_weaviate_cloud`` and ``use_async_with_custom`` all pin gRPC to the REST +endpoint under ``/v1/grpc-web``, because native gRPC is impossible there — the same +contract as the TypeScript ``@weaviate/web`` client. ``use_async_with_custom`` still +requires ``grpc_host``/``grpc_port``/``grpc_secure``; give it the HTTP values, or it +warns that it overrode them. An explicit ``import weaviate_client_web`` before ``import weaviate`` also works and remains the explicit form. The shim is installed automatically only under Emscripten, so diff --git a/packages/web/src/weaviate_client_web/_channel.py b/packages/web/src/weaviate_client_web/_channel.py index d6dd4b7cc..af899849c 100644 --- a/packages/web/src/weaviate_client_web/_channel.py +++ b/packages/web/src/weaviate_client_web/_channel.py @@ -339,10 +339,12 @@ def _no_path_prefix_hint() -> str: from weaviate.exceptions import GRPC_WEB_MIN_SERVER_VERSION, GRPC_WEB_SERVER_PATH_PREFIX return ( - "(no grpc_path_prefix set — for Weaviate >= " - f"{GRPC_WEB_MIN_SERVER_VERSION} use use_async_with_custom(..., " - f"grpc_path_prefix='{GRPC_WEB_SERVER_PATH_PREFIX}') on the REST port, or point " - "grpc_host/grpc_port at a grpc-web transcoder)" + "(no grpc_path_prefix set — under WebAssembly the connect helpers route gRPC to " + f"the REST endpoint under '{GRPC_WEB_SERVER_PATH_PREFIX}' by themselves, so use " + "one of them; hand-built ConnectionParams must set " + f"grpc_path_prefix='{GRPC_WEB_SERVER_PATH_PREFIX}' for Weaviate >= " + f"{GRPC_WEB_MIN_SERVER_VERSION}, or point grpc_host/grpc_port at a grpc-web " + "transcoder)" ) diff --git a/packages/web/tests/test_transport.py b/packages/web/tests/test_transport.py index 02871fa69..70777f10c 100644 --- a/packages/web/tests/test_transport.py +++ b/packages/web/tests/test_transport.py @@ -541,12 +541,12 @@ async def boom(url, headers, body, timeout): def test_unavailable_without_path_prefix_under_emscripten_hints_at_grpc_path_prefix(monkeypatch): - # use_async_with_local() under Pyodide builds a prefix-less channel aimed at the - # native gRPC port, which fetch cannot reach; the error must say what to do instead + # the connect helpers always set the prefix under Emscripten, so a prefix-less channel + # here means hand-built ConnectionParams; the error must say what to do instead details = _unavailable_details(monkeypatch, path_prefix="", platform="emscripten") assert "grpc_path_prefix='/v1/grpc-web'" in details assert "1.38.3" in details - assert "use_async_with_custom" in details + assert "connect helpers" in details def test_unavailable_with_path_prefix_has_no_prefix_hint(monkeypatch): diff --git a/test/test_connection_params.py b/test/test_connection_params.py index a0dc1c7d4..0527962af 100644 --- a/test/test_connection_params.py +++ b/test/test_connection_params.py @@ -1,3 +1,5 @@ +import sys + import pytest from pydantic import ValidationError @@ -134,22 +136,6 @@ def test_grpc_channel_rejects_prefix_for_sync_client() -> None: _grpc_web_params()._grpc_channel(proxies={}, grpc_msg_size=None, is_async=False) -def test_connect_to_custom_rejects_grpc_web_prefix() -> None: - # The synchronous helper must reject grpc-web up front (before connecting). - import weaviate - - with pytest.raises(WeaviateInvalidInputError, match="async-only"): - weaviate.connect_to_custom( - http_host="localhost", - http_port=8080, - http_secure=False, - grpc_host="localhost", - grpc_port=8080, - grpc_secure=False, - grpc_path_prefix="/grpc-web", - ) - - def test_grpc_channel_omits_option_without_prefix(monkeypatch) -> None: captured: dict = {} @@ -173,24 +159,6 @@ def fake_insecure_channel(target, options=None, **kwargs): assert "grpc-web.path_prefix" not in option_keys -def test_connect_to_custom_treats_root_prefix_as_native_grpc(monkeypatch) -> None: - # "/" normalizes to "" (native gRPC), so the async-only guard must not fire on it - import weaviate - import weaviate.connect.helpers as helpers - - monkeypatch.setattr(helpers, "__connect", lambda client: client) # module-level: not mangled - client = weaviate.connect_to_custom( - http_host="localhost", - http_port=8080, - http_secure=False, - grpc_host="localhost", - grpc_port=50051, - grpc_secure=False, - grpc_path_prefix="/", - ) - assert client._connection._connection_params._grpc_web_path_prefix == "" - - def test_async_client_construction_rejects_prefix_without_shim(monkeypatch) -> None: # fail at construction with actionable text, not deep inside connect() after the # OIDC and /v1/meta round trips already succeeded @@ -219,3 +187,132 @@ def test_sync_client_construction_rejects_grpc_web_prefix() -> None: with pytest.raises(WeaviateInvalidInputError, match="async"): WeaviateClient(_grpc_web_params()) + + +# --- the connect helpers' public surface --------------------------------------------- +# +# grpc-web is not something a caller selects: there is no grpc_path_prefix parameter on +# any helper, and off Emscripten the params they build are exactly what they always were. + + +def _params_of(client) -> ConnectionParams: + return client._connection._connection_params + + +@pytest.mark.parametrize( + "call", + [ + lambda w: w.use_async_with_local(grpc_path_prefix="/v1/grpc-web"), + lambda w: w.use_async_with_weaviate_cloud( + "abc.weaviate.cloud", None, grpc_path_prefix="/v1/grpc-web" + ), + lambda w: w.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=8080, + grpc_secure=False, + grpc_path_prefix="/v1/grpc-web", + ), + lambda w: w.connect_to_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=8080, + grpc_secure=False, + grpc_path_prefix="/v1/grpc-web", + ), + lambda w: w.connect_to_local(grpc_path_prefix="/v1/grpc-web"), + ], +) +def test_no_helper_takes_a_grpc_path_prefix(call) -> None: + # deliberate removal: grpc-web is chosen by the platform, never by the caller, so the + # parameter must not exist on any helper (the TypeScript web client has none either) + import weaviate + + with pytest.raises(TypeError, match="grpc_path_prefix"): + call(weaviate) + + +@pytest.mark.parametrize( + "call,expected", + [ + ( + lambda w: w.use_async_with_local(), + { + "http": {"host": "localhost", "port": 8080, "secure": False}, + "grpc": {"host": "localhost", "port": 50051, "secure": False}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_local(host="wv", port=9090, grpc_port=50052), + { + "http": {"host": "wv", "port": 9090, "secure": False}, + "grpc": {"host": "wv", "port": 50052, "secure": False}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None), + { + "http": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc": {"host": "grpc-abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_weaviate_cloud("abc.something.weaviate.network", None), + { + "http": {"host": "abc.something.weaviate.network", "port": 443, "secure": True}, + "grpc": { + "host": "abc.grpc.something.weaviate.network", + "port": 443, + "secure": True, + }, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=50051, + grpc_secure=False, + ), + { + "http": {"host": "localhost", "port": 8080, "secure": False}, + "grpc": {"host": "localhost", "port": 50051, "secure": False}, + "grpc_path_prefix": None, + }, + ), + ( + lambda w: w.use_async_with_custom( + http_host="rest.example.com", + http_port=443, + http_secure=True, + grpc_host="grpc.example.com", + grpc_port=443, + grpc_secure=True, + ), + { + "http": {"host": "rest.example.com", "port": 443, "secure": True}, + "grpc": {"host": "grpc.example.com", "port": 443, "secure": True}, + "grpc_path_prefix": None, + }, + ), + ], +) +def test_helper_params_off_emscripten_are_unchanged(call, expected) -> None: + # THE no-regression pin. The http/grpc values below are origin/main's verbatim; only + # grpc_path_prefix is new, and off Emscripten it is always None (native gRPC). + import weaviate + + assert sys.platform != "emscripten" + params = _params_of(call(weaviate)) + assert params.model_dump() == expected + assert params._grpc_web_path_prefix == "" diff --git a/test/test_wasm_compat.py b/test/test_wasm_compat.py index 5ac1d58fc..c2289970c 100644 --- a/test/test_wasm_compat.py +++ b/test/test_wasm_compat.py @@ -265,3 +265,148 @@ def test_deadlines_view_sanitises_every_timeout() -> None: assert deadlines.stream is None # the user's own config object is left untouched assert conn.timeout_config.init == float("inf") + + +# --- grpc-web auto-routing under Emscripten ------------------------------------------- +# +# Native gRPC is impossible under WASM (no sockets, no grpcio wheel), so the async connect +# helpers pin gRPC to the REST endpoint under Weaviate's own grpc-web base path — the same +# contract as the TypeScript @weaviate/web client's webify(). Nothing selects it. + +GRPC_WEB_PREFIX = "/v1/grpc-web" + + +@pytest.fixture +def emscripten(monkeypatch): + """Fake Emscripten, with the grpc-web shim marked active. + + Under real Pyodide ``import weaviate`` installs the shim itself; here only the + routing decision is under test, not the environment check that guards it. + """ + import weaviate.connect.base as base_mod + + monkeypatch.setattr(sys, "platform", "emscripten") + monkeypatch.setattr(base_mod.grpc, "__weaviate_client_web_shim__", True, raising=False) + + +def _params(client) -> ConnectionParams: + return client._connection._connection_params + + +def _assert_grpc_rides_rest(client) -> None: + params = _params(client) + assert params.grpc.model_dump() == params.http.model_dump() + assert params._grpc_web_path_prefix == GRPC_WEB_PREFIX + assert params._grpc_target == f"{params.http.host}:{params.http.port}" + + +def test_use_async_with_local_routes_grpc_to_rest_under_emscripten(emscripten) -> None: + import weaviate + + _assert_grpc_rides_rest(weaviate.use_async_with_local(host="localhost", port=8290)) + assert _params(weaviate.use_async_with_local()).model_dump() == { + "http": {"host": "localhost", "port": 8080, "secure": False}, + "grpc": {"host": "localhost", "port": 8080, "secure": False}, + "grpc_path_prefix": GRPC_WEB_PREFIX, + } + + +def test_use_async_with_weaviate_cloud_routes_grpc_to_the_cluster_host(emscripten) -> None: + # WCD serves grpc-web on the cluster's own REST endpoint, not on grpc- + import weaviate + + client = weaviate.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None) + _assert_grpc_rides_rest(client) + assert _params(client).model_dump() == { + "http": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc": {"host": "abc.something.weaviate.cloud", "port": 443, "secure": True}, + "grpc_path_prefix": GRPC_WEB_PREFIX, + } + + +def test_use_async_with_custom_routes_grpc_to_rest_under_emscripten(emscripten) -> None: + import weaviate + + _assert_grpc_rides_rest( + weaviate.use_async_with_custom( + http_host="wv.example.com", + http_port=443, + http_secure=True, + grpc_host="wv.example.com", + grpc_port=443, + grpc_secure=True, + ) + ) + + +def test_matching_grpc_arguments_are_not_warned_about(emscripten, recwarn) -> None: + # the documented WASM shape: gRPC arguments equal to the HTTP ones. Nothing is + # discarded, so warning here would just train users to ignore the warning. + import weaviate + + weaviate.use_async_with_custom( + http_host="localhost", + http_port=8290, + http_secure=False, + grpc_host="localhost", + grpc_port=8290, + grpc_secure=False, + ) + weaviate.use_async_with_local(port=8290) + weaviate.use_async_with_weaviate_cloud("abc.something.weaviate.cloud", None) + assert [str(w.message) for w in recwarn] == [] + + +def test_overridden_grpc_arguments_are_warned_about(emscripten) -> None: + # Python cannot drop required parameters the way TypeScript drops them from a type, + # so a WASM caller must pass something. Overriding keeps the client usable, but it + # must never look like the endpoint they gave was honoured. + import weaviate + + with pytest.warns(UserWarning, match="Con006") as record: + client = weaviate.use_async_with_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="grpc.example.com", + grpc_port=50051, + grpc_secure=True, + ) + msg = str(record[0].message) + assert "grpc.example.com:50051" in msg # what was discarded ... + assert "localhost:8080" in msg # ... and what is used instead + assert "WebAssembly" in msg # ... and why + _assert_grpc_rides_rest(client) + + +def test_an_explicit_local_grpc_port_is_warned_about_but_the_default_is_not(emscripten) -> None: + import weaviate + + with pytest.warns(UserWarning, match="Con006"): + client = weaviate.use_async_with_local(port=8080, grpc_port=8081) + _assert_grpc_rides_rest(client) + + +@pytest.mark.parametrize( + "call", + [ + lambda w: w.connect_to_local(), + lambda w: w.connect_to_weaviate_cloud( + "abc.something.weaviate.cloud", w.classes.init.Auth.api_key("k") + ), + lambda w: w.connect_to_custom( + http_host="localhost", + http_port=8080, + http_secure=False, + grpc_host="localhost", + grpc_port=50051, + grpc_secure=False, + ), + ], +) +def test_sync_helpers_still_raise_async_only_under_emscripten(emscripten, call) -> None: + # grpc-web routing must not have made the unsupported sync client look viable + import weaviate + + with pytest.raises(WeaviateStartUpError, match="async client"): + call(weaviate) diff --git a/weaviate/connect/helpers.py b/weaviate/connect/helpers.py index 28153ddc1..726767d0e 100644 --- a/weaviate/connect/helpers.py +++ b/weaviate/connect/helpers.py @@ -1,5 +1,6 @@ """Helper functions for creating new WeaviateClient or WeaviateAsyncClient instances in common scenarios.""" +import sys from typing import Dict, Optional, Tuple, Union from urllib.parse import urlparse @@ -17,11 +18,45 @@ from weaviate.config import AdditionalConfig from weaviate.connect.base import ConnectionParams, ProtocolParams from weaviate.embedded import WEAVIATE_VERSION, EmbeddedOptions -from weaviate.exceptions import WeaviateInvalidInputError +from weaviate.exceptions import GRPC_WEB_SERVER_PATH_PREFIX from weaviate.util import docstring_deprecated from weaviate.validator import _validate_input, _ValidateArgument from weaviate.warnings import _Warnings +# The native-gRPC port a local Weaviate exposes by default. Doubles as the sentinel for +# "the caller did not pick a gRPC port of their own" in use_async_with_local(). +_LOCAL_GRPC_PORT_DEFAULT = 50051 + + +def _webify( + http: ProtocolParams, grpc: ProtocolParams, *, grpc_chosen_by_caller: bool +) -> ConnectionParams: + """Build connection params, routing gRPC over grpc-web under WebAssembly. + + Under Emscripten there is no grpcio wheel and no socket, so native gRPC cannot work + at all; grpc-web on the REST listener is the only transport that can. gRPC is + therefore pinned to the HTTP endpoint under Weaviate's own grpc-web base path, which + is what the TypeScript ``@weaviate/web`` client does (its ``webify()``). Everywhere + else this is the identity: ``grpc`` is used exactly as given. + + ``grpc_chosen_by_caller`` says whether ``grpc`` came from the caller rather than from + a convention of the helper's own; discarding a caller's endpoint warns, so nobody is + left believing an endpoint was honoured when it was not. + """ + if sys.platform != "emscripten": + # grpc_path_prefix passed explicitly: it keeps the constructor arguments (and so + # pydantic's echo of them in a validation error) identical to what callers saw + # before grpc-web existed. + return ConnectionParams(http=http, grpc=grpc, grpc_path_prefix=None) + + web_grpc = ProtocolParams(host=http.host, port=http.port, secure=http.secure) + if grpc_chosen_by_caller and web_grpc != grpc: + _Warnings.grpc_endpoint_forced_to_grpc_web( + requested=f"{grpc.host}:{grpc.port}", + effective=f"{web_grpc.host}:{web_grpc.port}", + ) + return ConnectionParams(http=http, grpc=web_grpc, grpc_path_prefix=GRPC_WEB_SERVER_PATH_PREFIX) + def __parse_weaviate_cloud_cluster_url(cluster_url: str) -> Tuple[str, str]: _validate_input(_ValidateArgument([str], "cluster_url", cluster_url)) @@ -291,7 +326,6 @@ def connect_to_custom( additional_config: Optional[AdditionalConfig] = None, auth_credentials: Optional[AuthCredentials] = None, skip_init_checks: bool = False, - grpc_path_prefix: Optional[str] = None, ) -> WeaviateClient: """Connect to a Weaviate instance with custom connection parameters. @@ -314,11 +348,6 @@ def connect_to_custom( a bearer token, in which case use `weaviate.classes.init.Auth.bearer_token()`, a client secret, in which case use `weaviate.classes.init.Auth.client_credentials()` or a username and password, in which case use `weaviate.classes.init.Auth.client_password()`. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. - grpc_path_prefix: grpc-web base-path prefix. grpc-web is async-only, so it is NOT - supported by the synchronous ``connect_to_custom`` — passing a non-empty value - raises ``WeaviateInvalidInputError``. Use - ``use_async_with_custom(..., grpc_path_prefix=...)`` instead. Defaults to None - (native gRPC). Returns: The client connected to the instance with the required parameters set appropriately. @@ -351,23 +380,16 @@ def connect_to_custom( True >>> # The connection is automatically closed when the context is exited. """ - connection_params = ConnectionParams.from_params( - http_host=http_host, - http_port=http_port, - http_secure=http_secure, - grpc_host=grpc_host, - grpc_port=grpc_port, - grpc_secure=grpc_secure, - grpc_path_prefix=grpc_path_prefix, - ) - if connection_params._grpc_web_path_prefix: # normalized: "/" is native gRPC - raise WeaviateInvalidInputError( - "grpc_path_prefix enables grpc-web, which is async-only; use " - "use_async_with_custom(...) instead of connect_to_custom(...)" - ) return __connect( WeaviateClient( - connection_params, + ConnectionParams.from_params( + http_host=http_host, + http_port=http_port, + http_secure=http_secure, + grpc_host=grpc_host, + grpc_port=grpc_port, + grpc_secure=grpc_secure, + ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, additional_config=additional_config, @@ -398,6 +420,10 @@ def use_async_with_weaviate_cloud( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under WebAssembly/Pyodide gRPC runs over grpc-web on the cluster's own REST endpoint + (443/TLS) rather than the separate ``grpc-`` host, because native gRPC cannot work + there. Nothing to configure: the cluster serves grpc-web itself. + Args: cluster_url: The WCD cluster URL or hostname to connect to. Usually in the form: rAnD0mD1g1t5.something.weaviate.cloud auth_credentials: The credentials to use for authentication with your Weaviate instance. This can be an API key, in which case pass a string or use `weaviate.classes.init.Auth.api_key()`, @@ -434,9 +460,11 @@ def use_async_with_weaviate_cloud( """ cluster_url, grpc_host = __parse_weaviate_cloud_cluster_url(cluster_url) return WeaviateAsyncClient( - connection_params=ConnectionParams( + connection_params=_webify( http=ProtocolParams(host=cluster_url, port=443, secure=True), grpc=ProtocolParams(host=grpc_host, port=443, secure=True), + # the grpc- host is this helper's own convention, never caller input + grpc_chosen_by_caller=False, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, @@ -460,10 +488,15 @@ def use_async_with_local( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under WebAssembly/Pyodide gRPC runs over grpc-web on the REST listener, because native + gRPC cannot work there. ``grpc_port`` is then replaced by ``port``; if you passed a + ``grpc_port`` of your own it is discarded and a ``UserWarning`` says so. + Args: host: The host to use for the underlying REST and GraphQL API calls. port: The port to use for the underlying REST and GraphQL API calls. - grpc_port: The port to use for the underlying gRPC API. + grpc_port: The port to use for the underlying gRPC API. Ignored under + WebAssembly/Pyodide, where gRPC shares the REST ``port`` over grpc-web. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. @@ -500,9 +533,11 @@ def use_async_with_local( >>> # The connection is automatically closed when the context is exited. """ return WeaviateAsyncClient( - connection_params=ConnectionParams( + connection_params=_webify( http=ProtocolParams(host=host, port=port, secure=False), grpc=ProtocolParams(host=host, port=grpc_port, secure=False), + # the default port is this helper's convention; anything else was chosen + grpc_chosen_by_caller=grpc_port != _LOCAL_GRPC_PORT_DEFAULT, ), additional_headers=headers, additional_config=additional_config, @@ -601,7 +636,6 @@ def use_async_with_custom( additional_config: Optional[AdditionalConfig] = None, auth_credentials: Optional[AuthCredentials] = None, skip_init_checks: bool = False, - grpc_path_prefix: Optional[str] = None, ) -> WeaviateAsyncClient: """Create an async client object ready to connect to a Weaviate instance with custom connection parameters. @@ -611,24 +645,29 @@ def use_async_with_custom( Once you are done with the client you should call `client.close()` to close the connection and free up resources. Alternatively, you can use the client as a context manager in an `async with` statement, which will automatically open/close the connection when the context is entered/exited. See the examples below for details. + Under WebAssembly/Pyodide gRPC runs over grpc-web on the REST listener, because native + gRPC cannot work there (no sockets, no ``grpcio`` wheel). ``grpc_host``, ``grpc_port`` + and ``grpc_secure`` are then replaced by ``http_host``, ``http_port`` and + ``http_secure``; if what you passed differed, it is discarded and a ``UserWarning`` + names both endpoints. This mirrors the TypeScript ``@weaviate/web`` client, which + removes those three options from its API altogether. + Args: http_host: The host to use for the underlying REST and GraphQL API calls. http_port: The port to use for the underlying REST and GraphQL API calls. http_secure: Whether to use https for the underlying REST and GraphQL API calls. - grpc_host: The host to use for the underlying gRPC API. - grpc_port: The port to use for the underlying gRPC API. - grpc_secure: Whether to use a secure channel for the underlying gRPC API. + grpc_host: The host to use for the underlying gRPC API. Ignored under + WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. + grpc_port: The port to use for the underlying gRPC API. Ignored under + WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. + grpc_secure: Whether to use a secure channel for the underlying gRPC API. Ignored + under WebAssembly/Pyodide, where gRPC shares the REST endpoint over grpc-web. headers: Additional headers to include in the requests, e.g. API keys for Cloud vectorization. additional_config: This includes many additional, rarely used config options. use wvc.init.AdditionalConfig() to configure. auth_credentials: The credentials to use for authentication with your Weaviate instance. This can be an API key, in which case pass a string or use `weaviate.classes.init.Auth.api_key()`, a bearer token, in which case use `weaviate.classes.init.Auth.bearer_token()`, a client secret, in which case use `weaviate.classes.init.Auth.client_credentials()` or a username and password, in which case use `weaviate.classes.init.Auth.client_password()`. skip_init_checks: Whether to skip the initialization checks when connecting to Weaviate. - grpc_path_prefix: Optional base-path prefix for a grpc-web endpoint served on the - same host:port as REST (e.g. "/grpc-web"). When set, gRPC requests are sent - over grpc-web to ``://:/...`` and sharing - the REST host:port is allowed. Requires the ``weaviate-client-web`` - package. Defaults to None (native gRPC). Returns: The client connected to the instance with the required parameters set appropriately. @@ -665,14 +704,11 @@ def use_async_with_custom( >>> # The connection is automatically closed when the context is exited. """ return WeaviateAsyncClient( - ConnectionParams.from_params( - http_host=http_host, - http_port=http_port, - http_secure=http_secure, - grpc_host=grpc_host, - grpc_port=grpc_port, - grpc_secure=grpc_secure, - grpc_path_prefix=grpc_path_prefix, + _webify( + http=ProtocolParams(host=http_host, port=http_port, secure=http_secure), + grpc=ProtocolParams(host=grpc_host, port=grpc_port, secure=grpc_secure), + # all three gRPC arguments are required here, so they are always caller input + grpc_chosen_by_caller=True, ), auth_client_secret=__parse_auth_credentials(auth_credentials), additional_headers=headers, diff --git a/weaviate/exceptions.py b/weaviate/exceptions.py index 3f0873a32..0edd63580 100644 --- a/weaviate/exceptions.py +++ b/weaviate/exceptions.py @@ -361,7 +361,7 @@ def __init__( if code is StatusCode.UNIMPLEMENTED: reason = f"""The server did not route the grpc-web path '{grpc_path_prefix}' at {address}. Either: - the server is too old: grpc-web is served from Weaviate {GRPC_WEB_MIN_SERVER_VERSION} onwards, and this server reports {weaviate_version or "an unknown version"}, or -- `grpc_path_prefix` is wrong: Weaviate serves grpc-web at '{GRPC_WEB_SERVER_PATH_PREFIX}'. +- the grpc-web base path is wrong: Weaviate serves grpc-web at '{GRPC_WEB_SERVER_PATH_PREFIX}'. The connect helpers set it themselves; only hand-built ConnectionParams choose it (grpc_path_prefix). """ else: reason = f"""This error could be due to one of several reasons: @@ -374,7 +374,7 @@ def __init__( Weaviate {weaviate_version} makes use of a high-speed gRPC API as well as a REST API. Unfortunately, the gRPC health check against Weaviate could not be completed. -This client is configured for grpc-web (grpc_path_prefix='{grpc_path_prefix}'), which carries gRPC over the REST endpoint {address}; there is no separate gRPC port. +This client speaks grpc-web (base path '{grpc_path_prefix}'), which carries gRPC over the REST endpoint {address}; there is no separate gRPC port. {reason}{observed}""" super().__init__(msg) diff --git a/weaviate/warnings.py b/weaviate/warnings.py index a1dd3180b..b8046b544 100644 --- a/weaviate/warnings.py +++ b/weaviate/warnings.py @@ -336,6 +336,19 @@ def grpc_max_msg_size_not_found() -> None: stacklevel=1, ) + @staticmethod + def grpc_endpoint_forced_to_grpc_web(requested: str, effective: str) -> None: + warnings.warn( + message=f"""Con006: The gRPC endpoint you gave ({requested}) was overridden with {effective}. + + Under WebAssembly/Pyodide there is no socket and no grpcio wheel, so native gRPC cannot be used at all; + gRPC runs over grpc-web on the REST listener, which is the endpoint above. Pass gRPC arguments matching + the HTTP ones to silence this warning. A grpc-web transcoder on a separate endpoint is not reachable + through these helpers - build weaviate.connect.ConnectionParams yourself if you need one.""", + category=UserWarning, + stacklevel=1, + ) + @staticmethod def unknown_permission_encountered(permission: Any) -> None: warnings.warn(