From 537a479e51640ab40a4585511f1addd65320156f Mon Sep 17 00:00:00 2001 From: fffonion Date: Wed, 26 Aug 2026 20:06:16 +0800 Subject: [PATCH] feat(http): add bounded request and callable SSE clients --- Cargo.lock | 602 +++++++++++- Cargo.toml | 34 +- README.md | 2 + build.rs | 15 +- crates/rustscript/Cargo.toml | 1 + crates/rustscript/tests/alias_smoke.rs | 8 + docs/callable-runtime.md | 8 + docs/http-client.md | 222 +++++ src/builtins/mod.rs | 2 +- src/builtins/runtime/http/config.rs | 82 ++ src/builtins/runtime/http/mod.rs | 1158 ++++++++++++++++++++++++ src/builtins/runtime/http/policy.rs | 254 ++++++ src/builtins/runtime/http/request.rs | 960 ++++++++++++++++++++ src/builtins/runtime/http/sse.rs | 899 ++++++++++++++++++ src/builtins/runtime/mod.rs | 8 +- src/lib.rs | 4 +- src/vm/async_host/mod.rs | 3 + src/vm/async_host/stream.rs | 370 ++++++++ src/vm/host.rs | 49 +- src/vm/host_runtime.rs | 43 +- src/vm/host_stream_tests.rs | 1125 +++++++++++++++++++++++ src/vm/instance.rs | 4 + src/vm/invocation.rs | 6 +- src/vm/mod.rs | 34 +- tests/host_binding_generation_tests.rs | 65 ++ tests/http_feature_gating_tests.rs | 32 + tests/vm/http_host_tests.rs | 718 +++++++++++++++ tests/vm/http_sse_tests.rs | 1048 +++++++++++++++++++++ 28 files changed, 7728 insertions(+), 28 deletions(-) create mode 100644 docs/http-client.md create mode 100644 src/builtins/runtime/http/config.rs create mode 100644 src/builtins/runtime/http/mod.rs create mode 100644 src/builtins/runtime/http/policy.rs create mode 100644 src/builtins/runtime/http/request.rs create mode 100644 src/builtins/runtime/http/sse.rs create mode 100644 src/vm/async_host/stream.rs create mode 100644 src/vm/host_stream_tests.rs create mode 100644 tests/http_feature_gating_tests.rs create mode 100644 tests/vm/http_host_tests.rs create mode 100644 tests/vm/http_sse_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 6a3b5704..ce940f5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,12 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "base64" version = "0.22.1" @@ -271,6 +277,23 @@ version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d953932541249c91e3fa70a75ff1e52adc62979a2a8132145d4b9b3e6d1a9b6a" +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -340,6 +363,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -351,9 +383,50 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] [[package]] name = "gimli" @@ -415,6 +488,183 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -466,6 +716,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + [[package]] name = "log" version = "0.4.29" @@ -519,6 +775,12 @@ dependencies = [ "libc", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "once_cell" version = "1.21.4" @@ -538,7 +800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9676d58588b220f7af69d7aa86108042d2acaf21dd24c641a6d9ef3c4e193ba" dependencies = [ "pd-host-function 0.22.2", - "syn", + "syn 2.0.117", ] [[package]] @@ -548,7 +810,7 @@ dependencies = [ "pd-host-schema", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -559,7 +821,7 @@ checksum = "d9c941589fbbb839a40f7b80595d7b8f3742a8811268d787218f0c45c274d1f9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -567,7 +829,7 @@ name = "pd-host-schema" version = "0.1.0" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -581,19 +843,28 @@ dependencies = [ "cranelift-module", "cranelift-native", "futures-channel", + "futures-util", + "http-body-util", + "hyper", + "hyper-util", "libc", "paste", "pd-edge-abi", "pd-host-function 0.1.0", + "rcgen", "regex", "rt-format", "rusqlite", + "rustls", "rustyline", "self_cell", "serde", "serde_json", - "syn", + "syn 2.0.117", "tokio", + "tokio-rustls", + "url", + "webpki-roots", "windows-sys 0.59.0", ] @@ -614,6 +885,22 @@ dependencies = [ "serde_json", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -626,6 +913,21 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" version = "1.0.106" @@ -654,6 +956,19 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "regalloc2" version = "0.13.5" @@ -709,6 +1024,20 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rt-format" version = "0.3.1" @@ -752,6 +1081,40 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustscript" version = "0.1.0" @@ -814,7 +1177,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -846,6 +1209,12 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" @@ -868,6 +1237,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -879,12 +1254,63 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "target-lexicon" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tokio" version = "1.49.0" @@ -909,9 +1335,25 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -930,6 +1372,30 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -948,6 +1414,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -975,6 +1450,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -1072,6 +1556,44 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.56" @@ -1089,7 +1611,67 @@ checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1a779ea3..687ad230 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,17 @@ name = "vm" [features] default = ["runtime", "cli", "cranelift-jit"] runtime = [] -async = ["runtime", "dep:tokio"] +async = ["runtime", "dep:tokio", "dep:futures-util"] +http-client = [ + "async", + "dep:http-body-util", + "dep:hyper", + "dep:hyper-util", + "dep:rustls", + "dep:tokio-rustls", + "dep:url", + "dep:webpki-roots", +] sqlite = ["runtime", "dep:rusqlite"] edge-abi = [ "dep:edge_abi", @@ -63,8 +73,16 @@ cranelift-jit = { version = "0.129.1", optional = true } cranelift-module = { version = "0.129.1", optional = true } cranelift-native = { version = "0.129.1", optional = true } pd-host-function = { path = "./pd-host-function", version = "0.1.0" } +http-body-util = { version = "0.1", optional = true } +hyper = { version = "1", default-features = false, features = ["client", "http1"], optional = true } +hyper-util = { version = "0.1", default-features = false, features = ["tokio"], optional = true } +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"], optional = true } +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"], optional = true } +webpki-roots = { version = "1", optional = true } rusqlite = { version = "0.32", default-features = false, features = ["bundled", "hooks", "limits"], optional = true } -tokio = { version = "1", features = ["rt-multi-thread", "net", "time", "sync", "fs", "io-util", "process"], optional = true } +url = { version = "2", optional = true } +futures-util = { version = "0.3", optional = true } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "net", "time", "sync", "fs", "io-util", "process"], optional = true } edge_abi = { package = "pd-edge-abi", version = "0.1.1", default-features = false, optional = true } futures-channel = "0.3" paste = "1" @@ -82,6 +100,8 @@ windows-sys = { version = "0.59", features = ["Win32_System_Diagnostics_Debug", libc = "0.2" [dev-dependencies] +futures-util = "0.3" +rcgen = "0.13" syn = { version = "2", features = ["full"] } tokio = { version = "1", features = ["macros", "rt", "time", "sync"] } @@ -100,5 +120,15 @@ name = "host_context_arch_tests" path = "tests/host_context_arch_tests.rs" required-features = ["runtime"] +[[test]] +name = "http_host_tests" +path = "tests/vm/http_host_tests.rs" +required-features = ["runtime", "http-client"] + +[[test]] +name = "http_sse_tests" +path = "tests/vm/http_sse_tests.rs" +required-features = ["runtime", "http-client"] + [build-dependencies] syn = { version = "2", features = ["full"] } diff --git a/README.md b/README.md index e9a15a75..46f34dc5 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ The complete language, runtime, and implementation guides live on the [RustScrip - [RSS language](https://rustscript.org/docs/reference/rss/) - [Host functions](https://rustscript.org/docs/reference/host-functions/) - [Runtime controls and artifacts](https://rustscript.org/docs/reference/runtime-controls/) +- [Callable-driven HTTP client contract](docs/http-client.md) +- [Script call frames and callable values](docs/callable-runtime.md) - [Compiler frontend syntax and feature support](src/compiler/frontends/README.md) ## Crate usage diff --git a/build.rs b/build.rs index 686ab35b..0a0fad7d 100644 --- a/build.rs +++ b/build.rs @@ -166,7 +166,7 @@ fn main() { catalog.retain(|entry| !entry.source_name.starts_with("sqlite::")); } - let host_sources = vec![ + let mut host_sources = vec![ SourceSpec { path: "src/builtins/runtime/host.rs".to_string(), module: "host".to_string(), @@ -178,6 +178,19 @@ fn main() { category: SourceCategory::DefaultHost, }, ]; + if env::var_os("CARGO_FEATURE_HTTP_CLIENT").is_some() { + host_sources.push(SourceSpec { + path: "src/builtins/runtime/http/mod.rs".to_string(), + module: "http".to_string(), + category: SourceCategory::DefaultHost, + }); + + host_sources.push(SourceSpec { + path: "src/builtins/runtime/http/sse.rs".to_string(), + module: "http::sse".to_string(), + category: SourceCategory::DefaultHost, + }); + } let builtin_sources = builtin_source_specs(&namespaces); let core_sources = [SourceSpec { path: "src/builtins/runtime/core.rs".to_string(), diff --git a/crates/rustscript/Cargo.toml b/crates/rustscript/Cargo.toml index 66b33b20..414982b0 100644 --- a/crates/rustscript/Cargo.toml +++ b/crates/rustscript/Cargo.toml @@ -14,6 +14,7 @@ name = "rustscript" default = ["runtime", "cli", "cranelift-jit"] runtime = ["pd_vm_crate/runtime"] sqlite = ["pd_vm_crate/sqlite"] +http-client = ["runtime", "pd_vm_crate/http-client"] edge-abi = ["pd_vm_crate/edge-abi"] cli = ["pd_vm_crate/cli"] cranelift-jit = ["pd_vm_crate/cranelift-jit"] diff --git a/crates/rustscript/tests/alias_smoke.rs b/crates/rustscript/tests/alias_smoke.rs index 5b709560..0f6479e8 100644 --- a/crates/rustscript/tests/alias_smoke.rs +++ b/crates/rustscript/tests/alias_smoke.rs @@ -49,3 +49,11 @@ fn alias_exports_public_invocation_stream_contract() { message: "boom".to_string(), }); } + +#[cfg(feature = "http-client")] +#[test] +fn alias_http_client_includes_runtime_contract() { + fn accept_runtime_result(_result: rustscript::RuntimeResult<()>) {} + + accept_runtime_result(Ok(())); +} diff --git a/docs/callable-runtime.md b/docs/callable-runtime.md index 217c3668..6b95ea34 100644 --- a/docs/callable-runtime.md +++ b/docs/callable-runtime.md @@ -78,6 +78,14 @@ PDRC recordings preserve full execution-frame metadata. Callable environments us Polling drives execution and provides backpressure: at most one event item is buffered between polls, and the VM does not produce items while the consumer is not polling. `stream::emit` validates only the configured per-item value bound; sequence assignment, receipts, persistence, and delivery policy belong to the embedding. At most one invocation is active per VM, `Invocation::cancel(reason)` cancels with a typed `CancellationReason`, and the low-level `Vm::run` pump is unchanged for custom drivers. +## Callable-driven HTTP streams + +With the `http-client` feature, `http::client::request(request)` and `http::client::sse(request, on_event)` are script-facing host imports. SSE is a long-running ordinary host call. Its handler has the schema `fn(map) -> map`. The host produces one event, the VM runs one child callback frame, and the returned action controls continuation before another event can arrive at the VM boundary. + +The callback may yield or wait in an ordinary async host call. Existing frame machinery resumes the callback first and returns its final action to the suspended HTTP call. The network future does not own or enter the VM and is not polled while the callback is active, so at most one item remains unacknowledged and callback completion supplies backpressure. + +The buffered and SSE imports are independent capabilities. SSE exposes no script request IDs, handles, detached resources, `next`, or cancellation callables. Its complete event maps, action maps, terminal summaries, bounds, destination policy, and lifecycle contract are documented in [HTTP client callable contract](http-client.md). + ## Optimized backends Whole-program AOT and Trace JIT use the same builtin call path (static catalog IDs) for environment binding, native frame dispatch for `callvalue`, and prototype-direct native dispatch for `callscript`. Script-frame entry and return preserve frame-relative locals and typed continuations. diff --git a/docs/http-client.md b/docs/http-client.md new file mode 100644 index 00000000..eb9c35c5 --- /dev/null +++ b/docs/http-client.md @@ -0,0 +1,222 @@ +# HTTP client callable contract + +RustScript exposes buffered HTTP and SSE as bounded host imports. The SSE call keeps one ordinary host call active and invokes a script callable for each event; it exposes no response stream object. + +The embedding must configure destination policy and grant each available callable explicitly. HTTP configuration and capability bindings are snapshotted when a call is admitted, so later profile or configuration changes cannot widen an active connection. + +Both APIs described below are available with the `http-client` feature. + +## Capabilities and profiles + +The two imports are independent capabilities: + +- `http::client::request` +- `http::client::sse` + +Granting `http::client::request` does not grant `http::client::sse`. A restricted host-function profile must allow every imported callable used by the program. Profiles remain isolated: a grant or configuration in one VM/profile does not authorize another. + +Each available API is a host import gated by the HTTP client feature. The two-import contract does not consume or change static builtin IDs. See [Script call frames and callable values](callable-runtime.md) for callable execution and backend behavior. + +## Buffered requests + +```rust +use http; +use bytes; + +let response = http::client::request({ + "method": "POST", + "url": "https://example.test/v1/messages", + "headers": {"content-type": "application/json"}, + "body": bytes::from_utf8("{}"), +}); +``` + +`http::client::request(request)` accepts a map with: + +- `method`: one of `GET`, `POST`, `PUT`, `PATCH`, `DELETE`, `HEAD`, or `OPTIONS`; +- `url`: an `http` or `https` URL admitted by host policy; +- `headers`: an optional string-to-string map; +- `body`: optional bytes or a string. + +The response is buffered under the configured response-body limit and returned as: + +```rust +{ + "status": 200, + "headers": {"content-type": "application/json"}, + "body": bytes, + "url": "https://example.test/v1/messages", +} +``` + +`url` is the final validated URL after redirects. The request body, response body, response head, redirect count, concurrent connection count, connect phase, and total request duration are bounded. `Host`, `Content-Length`, `Transfer-Encoding`, and `Connection` are client-managed request headers. A limit, policy, transport, TLS, redirect, or timeout failure is a host error and produces no response map. + +## Server-sent events + +`http::client::sse` is available with the `http-client` feature. + +```rust +fn on_sse(item: map) -> map { + if item["kind"] == "event" { + print(item["data"]); + } + return {"action": "continue"}; +} + +let result = http::client::sse({ + "method": "GET", + "url": "https://example.test/events", + "headers": {"accept": "text/event-stream"}, +}, on_sse); +``` + +`http::client::sse(request, on_event)` uses this request map: + +| Field | Required | Accepted type and value | Bound or policy | +| --- | --- | --- | --- | +| `method` | yes | string: `GET` or `POST` | Other methods are rejected before transport admission | +| `url` | yes | string containing an `http` or `https` URL | Protocol family and the configured scheme, host, port, and address policy must all admit it | +| `headers` | no | map from string header names to string values | Names and values must be syntactically valid; client-managed request headers remain forbidden, and `Accept: text/event-stream` is supplied when absent | +| `body` | no | bytes or string, including for `POST` | Bounded by `max_request_body_bytes` | +| `timeout_ms` | no | positive integer milliseconds | Caps this optional shortening deadline by `HttpConfig::max_stream_duration` | + +The callback schema is `fn(map) -> map`, and the response must have an event-stream content type. The response head remains bounded by the existing HTTP parser. The contract adds no configurable request-header byte accounting. + +The callback receives exactly one map at a time, in this order: + +```rust +// The response was accepted; this precedes every event. +{ + "kind": "open", + "status": 200, + "headers": map, + "url": string, +} + +// One parsed event. "event" is per-dispatch state, reset to null at every +// dispatch boundary (including a blank line that dispatches no event); "id" +// and "retry_ms" are persistent stream state, retaining the last valid +// values seen so far and null only before any value has been seen. +{ + "kind": "event", + "event": string | null, + "data": string, + "id": string | null, + "retry_ms": int | null, +} + +// Clean EOF, after every preceding event callback completed. +{"kind": "end"} +``` + +The callback must return one of: + +```rust +{"action": "continue"} +{"action": "stop"} +``` + +`continue` acknowledges the item and permits the next network poll. `stop` ends the call locally. Any other shape or action is a callback error. + +SSE parsing follows the event-stream grammar: + +- UTF-8 text may start with one byte-order mark; +- `\r\n`, `\r`, and `\n` line endings are recognized; +- repeated `data:` fields are joined with `\n`, with the final join newline removed at dispatch; +- `event`, `id`, and decimal non-negative `retry` fields are normalized into the event map; +- comments and unknown fields are ignored; +- a blank line dispatches only after at least one `data:` field; +- malformed UTF-8, an over-limit line or event, and cumulative received event-stream application bytes exceeding the call limit are host errors. + +There is no automatic reconnection. Values such as a provider's `[DONE]` marker remain ordinary event data. + +## Terminal summaries and errors + +After callback processing terminates normally, the SSE call returns one summary: + +```rust +{ + "outcome": "eof" | "stopped", + "status": int, + "headers": map, + "url": string, + "items": int, + "bytes_received": int, + "bytes_sent": int, +} +``` + +`items` counts delivered callback items. `bytes_received` and `bytes_sent` are observational summary counters. Limit enforcement uses independent entire-call accounting and does not depend on whether or how these counters are displayed. + +Transport, parser, destination-policy, timeout, and callback failures stay errors. They are never converted into a successful terminal summary. + +## Sequencing, backpressure, and lifecycle + +Streaming is a single caller-owned operation: + +1. the host polls for one protocol item; +2. the VM invokes `on_event` in a child script frame; +3. the callback returns one action; +4. the host applies that action before polling for another item. + +At most one unacknowledged protocol item crosses the host/VM boundary. Decoder scratch space is bounded separately. The network future is not polled while the callback runs, yields, or waits in another async host call. If the callback yields or invokes an ordinary async host function, the callback resumes first; only its final action resumes the outer stream operation. This sequencing supplies backpressure without a background reader or callback queue. + +The network future never owns or re-enters the VM. Callback error, protocol completion, configured deadline, VM reset/shutdown/drop, invocation termination, or normal return retires the operation exactly once. The embedding owns pending futures: retiring a call drops its transport and permit, and a late completion cannot re-enter the VM. + +`request_timeout` is the total bound for a buffered request and does not apply to SSE. `max_stream_duration` is the host-controlled absolute total-duration bound for each SSE call. SSE computes one admission-time deadline from the smaller of `max_stream_duration` and optional positive `timeout_ms`; the script value can only shorten the call and cannot disable or extend the host maximum. DNS, TCP, TLS, active reads, callback execution, and callback waits all count against the same deadline. Embedding invocation retirement may terminate the call sooner. `stream_idle_timeout` remains a separate wait-for-network-progress bound and resets only after progress; periodic traffic cannot extend the total deadline. Network idle time excludes time spent inside the callback, while callback work remains inside the total deadline. + +## Configuration defaults + +`HttpConfig` uses explicit bounded defaults. Streaming byte limits and all timeout fields must remain positive: + +| Field | Default | Purpose | +| --- | ---: | --- | +| `allowed_schemes` | `https` | Scheme allowlist; protocol-family checks still apply | +| `allowed_hosts` | empty | Destination host allowlist; empty denies every host | +| `allowed_ports` | empty | Destination port allowlist; empty denies every port | +| `allow_private_ips` | `false` | Reject private and other special-use addresses | +| `max_redirects` | 5 | Buffered/SSE redirect bound | +| `max_request_body_bytes` | 1 MiB | Request body bound | +| `max_response_body_bytes` | 8 MiB | Buffered response body bound | +| `connect_timeout` | 10 s | DNS/connect/TLS phase bound | +| `request_timeout` | 30 s | Buffered request total duration | +| `max_stream_item_bytes` | 1 MiB | SSE event bound | +| `max_stream_total_bytes` | 64 MiB | Entire-call cumulative received event-stream byte bound | +| `max_sse_line_bytes` | 64 KiB | SSE line bound | +| `max_stream_duration` | 5 min | Host maximum total duration for SSE calls | +| `stream_idle_timeout` | 30 s | Wait-for-network-data bound | + +The shared in-flight connection default is 64. Zero values for streaming byte limits or any timeout are invalid configuration; buffered `max_request_body_bytes` and `max_response_body_bytes` may be zero to prohibit request or response payload bytes. `HttpConfig::default()` allows `https`. Embeddings should set explicit host and port allowlists and add `http` only when cleartext transport is required. Buffered HTTP and SSE accept only `http`/`https`. + +## Destination policy and protocol transports + +Every protocol uses the same admission, address-pinning, and security policy: + +- URLs require a host and reject userinfo; +- both the protocol's scheme family and the configured scheme allowlist must admit the URL; +- host and effective port must match their configured allowlists; +- every DNS result is validated, and the selected validated address is pinned for the connection; +- when private addresses are disabled, private, loopback, link-local, multicast, unspecified, documentation, transition, reserved, and other special-use IPv4/IPv6 ranges are rejected; IPv4-mapped IPv6 addresses receive the IPv4 checks; +- the original validated hostname remains the TLS SNI name and HTTP `Host` authority when connecting to a pinned address; +- buffered HTTP and SSE revalidate every redirect and remove `Authorization` and `Cookie` on a cross-origin redirect; +- ambient proxy settings are ignored. There is no implicit cookie jar, authentication source, or global proxy state. + +The policy snapshot taken at call admission applies for the complete operation. + +Buffered HTTP and SSE use direct Hyper HTTP/1 over Tokio/Rustls connections and perform no independent DNS lookup outside the shared admission and pinning path. + +## Deliberately absent APIs and semantics + +RustScript core provides no script-visible HTTP request ID, response/stream handle, `next`, `next_event`, or `cancel` callable. Streams cannot detach from their caller. There is no multiplexing, background reader, automatic reconnect, provider/model interpretation, agent loop, or platform retry policy. Applications implement provider-specific JSON, `[DONE]`, tool-call deltas, retry rules, and reconnect decisions in RSS or downstream hosts. + +## Cancellation migration + +PR #13 introduced HTTP-private pending-operation and abort-handle maps, one abort pair per request, HTTP owner routes, request-local runtimes, and HTTP-synthesized cancellation errors. The callable streaming contract supersedes those mechanisms. Buffered requests and SSE submit ordinary futures through the embedding-owned async bridge; HTTP has no private pending map, abort map, operation-ID namespace, token owner route, or cancellation state machine. + +The generic `src/builtins/runtime/cancellation.rs` remains for non-HTTP runtime callers. HTTP does not depend on `CancellationToken`, `CancellationReason`, `OperationOwner::Http`, or owner-wide cancellation routing. Embedding-owned retirement of a pending future remains VM lifecycle control and rejects late completion; dropping an `Invocation` also retires active producer/callback waits and returns the VM and connection permit for reuse. This lifecycle cleanup is not an HTTP API-level cancellation facility. + +## Target and backend notes + +The callable pump follows the ordinary host-call suspension boundary for interpreter, Trace JIT, and whole-program AOT execution. Network futures remain outside VM execution, and callback frames use the same wait/yield continuation rules across backends. + +`pd-vm-nostd` retains callable metadata and static builtin IDs without including HTTP transport implementations. WebAssembly and other embeddings can expose host imports only when that embedding supplies the capability, policy configuration, and async driving required by this contract. The contract does not imply an HTTP backend on targets where the host has not provided one. diff --git a/src/builtins/mod.rs b/src/builtins/mod.rs index 761ab5a2..780f8ebb 100644 --- a/src/builtins/mod.rs +++ b/src/builtins/mod.rs @@ -5,7 +5,7 @@ mod metadata; #[cfg(feature = "runtime")] pub(crate) mod runtime; -#[cfg(test)] +#[cfg_attr(not(feature = "http-client"), allow(unused_imports))] pub use self::metadata::CallableType; pub use self::metadata::{ CallableDef, CallableParam, CallableParamType, CallableSignature, HostExecution, diff --git a/src/builtins/runtime/http/config.rs b/src/builtins/runtime/http/config.rs new file mode 100644 index 00000000..42b9c2d4 --- /dev/null +++ b/src/builtins/runtime/http/config.rs @@ -0,0 +1,82 @@ +use std::time::Duration; + +use crate::vm::{VmError, VmResult}; + +/// Bounded network policy for the built-in HTTP client and future streaming adapters. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HttpConfig { + pub allowed_schemes: Vec, + pub allowed_hosts: Vec, + pub allowed_ports: Vec, + pub max_redirects: usize, + pub max_request_body_bytes: usize, + pub max_response_body_bytes: usize, + pub connect_timeout: Duration, + pub request_timeout: Duration, + pub allow_private_ips: bool, + pub max_stream_item_bytes: usize, + pub max_stream_total_bytes: usize, + pub max_sse_line_bytes: usize, + pub max_stream_duration: Duration, + pub stream_idle_timeout: Duration, +} + +impl HttpConfig { + /// Validates limits that must remain positive for every streaming adapter. + pub fn validate(&self) -> VmResult<()> { + let positive_limits = [ + ("max_stream_item_bytes", self.max_stream_item_bytes), + ("max_stream_total_bytes", self.max_stream_total_bytes), + ("max_sse_line_bytes", self.max_sse_line_bytes), + ]; + if let Some((name, _)) = positive_limits.iter().find(|(_, value)| *value == 0) { + return Err(VmError::HostError(format!( + "HTTP configuration field '{name}' must be positive" + ))); + } + let positive_timeouts = [ + ("connect_timeout", self.connect_timeout), + ("request_timeout", self.request_timeout), + ("max_stream_duration", self.max_stream_duration), + ("stream_idle_timeout", self.stream_idle_timeout), + ]; + if let Some((name, _)) = positive_timeouts + .iter() + .find(|(_, timeout)| timeout.is_zero()) + { + return Err(VmError::HostError(format!( + "HTTP configuration field '{name}' must be positive" + ))); + } + if let Some((name, _)) = positive_timeouts + .iter() + .find(|(_, timeout)| std::time::Instant::now().checked_add(*timeout).is_none()) + { + return Err(VmError::HostError(format!( + "HTTP configuration field '{name}' is too large" + ))); + } + Ok(()) + } +} + +impl Default for HttpConfig { + fn default() -> Self { + Self { + allowed_schemes: vec!["https".to_string()], + allowed_hosts: Vec::new(), + allowed_ports: Vec::new(), + max_redirects: 5, + max_request_body_bytes: 1024 * 1024, + max_response_body_bytes: 8 * 1024 * 1024, + connect_timeout: Duration::from_secs(10), + request_timeout: Duration::from_secs(30), + allow_private_ips: false, + max_stream_item_bytes: 1024 * 1024, + max_stream_total_bytes: 64 * 1024 * 1024, + max_sse_line_bytes: 64 * 1024, + max_stream_duration: Duration::from_secs(5 * 60), + stream_idle_timeout: Duration::from_secs(30), + } + } +} diff --git a/src/builtins/runtime/http/mod.rs b/src/builtins/runtime/http/mod.rs new file mode 100644 index 00000000..397565fc --- /dev/null +++ b/src/builtins/runtime/http/mod.rs @@ -0,0 +1,1158 @@ +use std::time::{Duration, Instant}; + +use pd_host_function::pd_host_function; + +use super::{borrow_arg, take_arg}; +use crate::builtins::runtime::VmMap; +use crate::vm::{CaptureAsyncHostContext, Vm, VmError, VmResult}; + +mod config; +pub(super) mod policy; +pub(super) mod request; +pub(super) mod sse; + +pub use config::HttpConfig; +use policy::{ConnectionAdmission, ConnectionPermit}; + +const DEFAULT_MAX_HTTP_IN_FLIGHT: usize = 64; + +#[derive(Clone)] +pub(crate) struct HttpHostState { + config: Option, + admission: ConnectionAdmission, +} + +/// HTTP host configuration owned by the HTTP host implementation. +pub trait HttpHostExt { + fn configure_http(&mut self, config: HttpConfig) -> VmResult<()>; + fn set_http_max_in_flight(&mut self, max_in_flight: usize); + fn http_max_in_flight(&self) -> usize; + fn clear_http_configuration(&mut self); + fn http_is_configured(&self) -> bool; +} + +impl HttpHostExt for Vm { + fn configure_http(&mut self, config: HttpConfig) -> VmResult<()> { + config.validate()?; + let admission = self + .host + .host_function_state::() + .map_or_else( + || ConnectionAdmission::new(DEFAULT_MAX_HTTP_IN_FLIGHT), + |state| state.admission.clone(), + ); + self.host.set_host_function_state(HttpHostState { + config: Some(config), + admission, + }); + Ok(()) + } + + fn set_http_max_in_flight(&mut self, max_in_flight: usize) { + if self.host.host_function_state::().is_none() { + self.host.set_host_function_state(HttpHostState { + config: None, + admission: ConnectionAdmission::new(DEFAULT_MAX_HTTP_IN_FLIGHT), + }); + } + self.host + .host_function_state_mut::() + .expect("HTTP host state was inserted") + .admission + .set_max_in_flight(max_in_flight); + } + + fn http_max_in_flight(&self) -> usize { + self.host + .host_function_state::() + .map_or(DEFAULT_MAX_HTTP_IN_FLIGHT, |state| { + state.admission.max_in_flight() + }) + } + + fn clear_http_configuration(&mut self) { + if let Some(state) = self.host.host_function_state_mut::() { + state.config = None; + } + } + + fn http_is_configured(&self) -> bool { + self.host + .host_function_state::() + .and_then(|state| state.config.as_ref()) + .is_some() + } +} + +pub(super) struct HttpRequestContext { + config: HttpConfig, + _permit: ConnectionPermit, +} + +impl HttpRequestContext { + fn capture_stream( + vm: &mut Vm, + script_timeout: Option, + protocol: &str, + ) -> VmResult<(Self, Instant)> { + let state = vm + .host + .host_function_state::() + .ok_or_else(|| VmError::HostError("HTTP host is not configured".to_string()))?; + let config = state + .config + .clone() + .ok_or_else(|| VmError::HostError("HTTP host is not configured".to_string()))?; + let admitted_at = Instant::now(); + if script_timeout.is_some_and(|timeout| admitted_at.checked_add(timeout).is_none()) { + return Err(VmError::HostError(format!( + "{protocol} timeout_ms cannot form a deadline" + ))); + } + let duration = script_timeout.map_or(config.max_stream_duration, |timeout| { + timeout.min(config.max_stream_duration) + }); + let deadline = admitted_at.checked_add(duration).ok_or_else(|| { + VmError::HostError("HTTP max_stream_duration cannot form a deadline".to_string()) + })?; + let permit = state.admission.acquire()?; + Ok(( + Self { + config, + _permit: permit, + }, + deadline, + )) + } +} + +impl CaptureAsyncHostContext for HttpRequestContext { + fn capture(vm: &mut Vm) -> VmResult { + let state = vm + .host + .host_function_state::() + .ok_or_else(|| VmError::HostError("HTTP host is not configured".to_string()))?; + let config = state + .config + .clone() + .ok_or_else(|| VmError::HostError("HTTP host is not configured".to_string()))?; + let permit = state.admission.acquire()?; + Ok(Self { + config, + _permit: permit, + }) + } +} + +/// Starts an HTTP request under the VM's configured network policy. +/// +/// The request map accepts `method`, `url`, optional `headers`, and optional `body`. +/// The response map contains `status`, `headers`, `body`, and the final `url`. +#[pd_host_function(name = "http::client::request")] +pub(super) async fn builtin_http_client_request( + #[pd_host_context] context: HttpRequestContext, + request: VmMap, +) -> VmResult { + request::perform_buffered_request(context, request).await +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + use std::time::{Duration, Instant}; + + use super::policy::{ + SchemeFamily, is_restricted_ip, request_deadline, validate_resolved_addresses, + validate_url, validate_url_policy, + }; + use super::request::{ + HttpRequest, ResponseReadObserver, execute_request, execute_request_with_observer, + execute_request_with_tls_config, pending_connection_test, + }; + use super::{HttpConfig, HttpHostExt, HttpRequestContext, builtin_http_client_request}; + use crate::builtins::runtime::VmMap; + use crate::vm::{ + CallOutcome, CallReturn, HostAsyncBridge, HostFuture, HostOpId, Value, VmResult, + }; + + #[test] + fn default_http_policy_denies_all_hosts() { + let config = HttpConfig::default(); + assert_eq!(config.allowed_schemes, ["https"]); + assert!(config.allowed_hosts.is_empty()); + assert!(config.allowed_ports.is_empty()); + assert!(!config.allow_private_ips); + config.validate().expect("default bounds should be valid"); + } + + #[test] + fn stream_timeout_validation_precedes_permit_admission() { + let mut vm = crate::vm::Vm::new(crate::vm::Program::new(Vec::new(), Vec::new())); + vm.set_http_max_in_flight(0); + vm.configure_http(HttpConfig::default()) + .expect("default config should be valid"); + + let error = HttpRequestContext::capture_stream(&mut vm, Some(Duration::MAX), "SSE") + .err() + .expect("an unrepresentable script timeout should be rejected"); + assert!(error.to_string().contains("timeout_ms"), "{error}"); + assert!( + !error.to_string().contains("in-flight request limit"), + "deadline validation must happen before permit admission: {error}" + ); + } + + #[test] + fn http_scheme_family_rejects_non_http_schemes() { + let config = HttpConfig { + allowed_schemes: vec!["http".into(), "https".into(), "ftp".into()], + allowed_hosts: vec!["example.com".into()], + allowed_ports: vec![80, 443], + ..HttpConfig::default() + }; + let http: url::Url = "https://example.com/".parse().expect("valid URL"); + let ftp: url::Url = "ftp://example.com/".parse().expect("valid URL"); + assert!(validate_url_policy(&config, SchemeFamily::Http, &http).is_ok()); + assert!(validate_url_policy(&config, SchemeFamily::Http, &ftp).is_err()); + } + + #[test] + fn request_submits_future_to_host_driver_without_runtime_operation() { + use std::task::{Context, Poll}; + + struct RecordingBridge { + submitted: Arc>>, + } + + impl HostAsyncBridge for RecordingBridge { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + *self.submitted.lock().expect("submission lock") = Some((op_id, future)); + Ok(()) + } + + fn poll_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + } + + let submitted = Arc::new(Mutex::new(None)); + let mut vm = crate::vm::Vm::new(crate::vm::Program::new(Vec::new(), Vec::new())); + vm.configure_http(HttpConfig::default()) + .expect("default config should be valid"); + vm.set_async_bridge(Box::new(RecordingBridge { + submitted: Arc::clone(&submitted), + })); + let args = [Value::Map(Arc::new(VmMap::default()))]; + + let outcome = builtin_http_client_request(&mut vm, &args) + .expect("HTTP async host call should submit"); + let CallOutcome::Pending(op_id) = outcome else { + panic!("HTTP async host call should suspend"); + }; + assert_eq!(op_id, 1); + assert_eq!( + submitted + .lock() + .expect("submission lock") + .as_ref() + .map(|(submitted_id, _)| *submitted_id), + Some(op_id) + ); + assert_eq!(vm.execution_scope().operations().active_count(), 0); + } + + #[test] + fn production_request_timeout_covers_delayed_headers() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have address"); + let server = std::thread::spawn(move || { + let (_socket, _) = listener.accept().expect("request should connect"); + std::thread::sleep(Duration::from_millis(100)); + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + connect_timeout: Duration::from_millis(50), + request_timeout: Duration::from_millis(20), + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("http://{address}/").parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + let error = runtime + .block_on(super::policy::with_deadline( + request_deadline(config.request_timeout).expect("valid request deadline"), + execute_request(&config, &request), + )) + .expect_err("hanging server should time out"); + assert!(error.to_string().contains("deadline exceeded")); + server.join().expect("server should exit"); + } + + #[test] + fn response_body_timeout_uses_the_same_total_deadline() { + use std::io::{Read, Write}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have address"); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().expect("request should connect"); + let mut request = [0u8; 1024]; + let _ = socket + .read(&mut request) + .expect("request should be readable"); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n") + .expect("headers should be written"); + socket.flush().expect("headers should flush"); + std::thread::sleep(Duration::from_millis(100)); + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + connect_timeout: Duration::from_millis(50), + request_timeout: Duration::from_millis(20), + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("http://{address}/").parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + let error = runtime + .block_on(super::policy::with_deadline( + request_deadline(config.request_timeout).expect("valid request deadline"), + execute_request(&config, &request), + )) + .expect_err("stalled response body should time out"); + assert!(error.to_string().contains("deadline exceeded")); + server.join().expect("server should exit"); + } + + #[test] + fn redirects_revalidate_policy_and_strip_cross_origin_credentials() { + use std::io::{Read, Write}; + + let first = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let first_address = first.local_addr().expect("listener should have address"); + let second = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let second_address = second.local_addr().expect("listener should have address"); + let first_server = std::thread::spawn(move || { + let (mut socket, _) = first.accept().expect("request should connect"); + let mut bytes = [0_u8; 2048]; + let read = socket.read(&mut bytes).expect("request should be readable"); + let request = String::from_utf8_lossy(&bytes[..read]).to_ascii_lowercase(); + assert!(request.contains("authorization: bearer secret")); + assert!(request.contains("cookie: session=secret")); + write!( + socket, + "HTTP/1.1 302 Found\r\nLocation: http://{second_address}/final\r\nContent-Length: 0\r\n\r\n" + ) + .expect("redirect should be writable"); + }); + let second_server = std::thread::spawn(move || { + let (mut socket, _) = second.accept().expect("request should connect"); + let mut bytes = [0_u8; 2048]; + let read = socket.read(&mut bytes).expect("request should be readable"); + let request = String::from_utf8_lossy(&bytes[..read]).to_ascii_lowercase(); + assert!(!request.contains("authorization:")); + assert!(!request.contains("cookie:")); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .expect("response should be writable"); + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![first_address.port(), second_address.port()], + allow_private_ips: true, + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("http://{first_address}/") + .parse() + .expect("valid URL"), + headers: vec![ + ( + hyper::header::AUTHORIZATION, + hyper::header::HeaderValue::from_static("Bearer secret"), + ), + ( + hyper::header::COOKIE, + hyper::header::HeaderValue::from_static("session=secret"), + ), + ], + body: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + let response = runtime + .block_on(super::policy::with_deadline( + request_deadline(config.request_timeout).expect("valid request deadline"), + execute_request(&config, &request), + )) + .expect("redirected request should complete"); + assert_eq!( + response.get(&Value::string("status")), + Some(&Value::Int(200)) + ); + assert_eq!( + response.get(&Value::string("url")), + Some(&Value::string(format!("http://{second_address}/final"))) + ); + first_server.join().expect("first server should exit"); + second_server.join().expect("second server should exit"); + } + + #[test] + fn redirect_destination_is_revalidated_before_connection() { + use std::io::{Read, Write}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have address"); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().expect("request should connect"); + let mut request = [0_u8; 1024]; + let _ = socket + .read(&mut request) + .expect("request should be readable"); + write!( + socket, + "HTTP/1.1 302 Found\r\nLocation: http://localhost:{}/blocked\r\nContent-Length: 0\r\n\r\n", + address.port() + ) + .expect("redirect should be writable"); + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("http://{address}/").parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + let error = runtime + .block_on(super::policy::with_deadline( + request_deadline(config.request_timeout).expect("valid request deadline"), + execute_request(&config, &request), + )) + .expect_err("redirect target should be denied"); + assert!(error.to_string().contains("target host is not allowed")); + server.join().expect("server should exit"); + } + + fn assert_redirect_userinfo_is_rejected(userinfo: &str) { + use std::io::{Read, Write}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have address"); + let userinfo = userinfo.to_string(); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().expect("first request should connect"); + let mut request = [0_u8; 2048]; + let read = socket + .read(&mut request) + .expect("request should be readable"); + let request = String::from_utf8_lossy(&request[..read]).to_ascii_lowercase(); + assert!(!request.contains("authorization:")); + assert!(!request.contains(&userinfo.to_ascii_lowercase())); + write!( + socket, + "HTTP/1.1 302 Found\r\nLocation: http://{userinfo}@{address}/blocked\r\nContent-Length: 0\r\n\r\n" + ) + .expect("redirect should be writable"); + drop(socket); + + listener + .set_nonblocking(true) + .expect("listener should become nonblocking"); + let deadline = Instant::now() + Duration::from_millis(200); + loop { + match listener.accept() { + Ok(_) => panic!("redirect userinfo must be rejected before a second request"), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("unexpected accept error: {error}"), + } + } + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("http://{address}/").parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + let error = runtime + .block_on(super::policy::with_deadline( + request_deadline(config.request_timeout).expect("valid request deadline"), + execute_request(&config, &request), + )) + .expect_err("redirect userinfo should be denied"); + assert!(error.to_string().contains("URL userinfo is not allowed")); + server.join().expect("server should exit"); + } + + #[test] + fn redirect_username_is_rejected_before_a_second_request() { + assert_redirect_userinfo_is_rejected("redirect-user"); + } + + #[test] + fn redirect_username_and_password_are_rejected_before_a_second_request() { + assert_redirect_userinfo_is_rejected("redirect-user:redirect-password"); + } + + fn execute_fixture_response_fragments_for( + method: hyper::Method, + response: Vec<&'static [u8]>, + max_response_body_bytes: usize, + ) -> (VmResult, ResponseReadObserver) { + use std::io::{Read, Write}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have address"); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().expect("request should connect"); + let mut request = [0_u8; 2048]; + let _ = socket + .read(&mut request) + .expect("request should be readable"); + for fragment in response { + socket + .write_all(fragment) + .expect("response fragment should be writable"); + socket.flush().expect("response fragment should flush"); + } + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + max_response_body_bytes, + ..HttpConfig::default() + }; + let request = HttpRequest { + method, + url: format!("http://{address}/").parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + let observer = ResponseReadObserver::default(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + let result = runtime.block_on(async { + tokio::time::timeout( + Duration::from_millis(500), + execute_request_with_observer(&config, &request, observer.clone()), + ) + .await + .expect("fixture response must make progress without deadline fallback") + }); + server.join().expect("server should exit"); + (result, observer) + } + + fn execute_fixture_response_fragments( + response: Vec<&'static [u8]>, + max_response_body_bytes: usize, + ) -> (VmResult, ResponseReadObserver) { + execute_fixture_response_fragments_for( + hyper::Method::GET, + response, + max_response_body_bytes, + ) + } + + fn execute_fixture_response( + response: &'static [u8], + max_response_body_bytes: usize, + ) -> (VmResult, ResponseReadObserver) { + execute_fixture_response_fragments(vec![response], max_response_body_bytes) + } + + #[test] + fn continue_then_final_response_in_one_write_reaches_the_final_head() { + let (result, observer) = execute_fixture_response( + b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + 8, + ); + let response = result.expect("final response should complete after 100 Continue"); + assert_eq!( + response.get(&Value::string("status")), + Some(&Value::Int(200)) + ); + assert_eq!( + response.get(&Value::string("body")), + Some(&Value::bytes(b"ok".to_vec())) + ); + assert!(observer.body_read_calls() > 0); + } + + #[test] + fn fragmented_continue_then_final_response_reaches_the_final_head() { + let (result, _) = execute_fixture_response_fragments( + vec![ + b"HTTP/1.1 100 Cont", + b"inue\r\n", + b"X-Info: yes\r\n\r", + b"\nHTTP/1.1 200 O", + b"K\r\nContent-Length: 2\r\n\r\n", + b"ok", + ], + 8, + ); + let response = result.expect("fragmented final response should complete after 100"); + assert_eq!( + response.get(&Value::string("body")), + Some(&Value::bytes(b"ok".to_vec())) + ); + } + + #[test] + fn early_hints_then_final_response_in_one_write_reaches_the_final_head() { + let (result, _) = execute_fixture_response( + b"HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok", + 8, + ); + let response = result.expect("final response should complete after 103 Early Hints"); + assert_eq!( + response.get(&Value::string("status")), + Some(&Value::Int(200)) + ); + } + + #[test] + fn fragmented_early_hints_then_final_response_reaches_the_final_head() { + let (result, _) = execute_fixture_response_fragments( + vec![ + b"HTTP/1.1 103 Early Hints\r\n", + b"Link: \r\n\r\nHTTP/1.1 ", + b"200 OK\r\nContent-Length: 2\r\n", + b"\r\nok", + ], + 8, + ); + let response = result.expect("fragmented final response should complete after 103"); + assert_eq!( + response.get(&Value::string("body")), + Some(&Value::bytes(b"ok".to_vec())) + ); + } + + fn tls_fixture_configs() -> (Arc, Arc) { + let certified = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]) + .expect("test certificate should generate"); + let cert_der = certified.cert.der().clone(); + let key_der = + rustls::pki_types::PrivateKeyDer::Pkcs8(certified.key_pair.serialize_der().into()); + let mut server_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![cert_der.clone()], key_der) + .expect("test server certificate should configure"); + server_config.alpn_protocols = vec![b"http/1.1".to_vec()]; + + let mut roots = rustls::RootCertStore::empty(); + roots + .add(cert_der) + .expect("test certificate should be trusted"); + let client_config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + assert!(client_config.alpn_protocols.is_empty()); + (Arc::new(server_config), Arc::new(client_config)) + } + + #[test] + fn https_requires_http11_alpn_and_preserves_sni_host_and_query() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + let (server_config, client_config) = tls_fixture_configs(); + let listener = runtime + .block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) + .expect("TLS listener should bind"); + let address = listener.local_addr().expect("TLS listener address"); + let server = runtime.spawn(async move { + let (stream, _) = listener.accept().await.expect("TLS request should connect"); + let mut stream = tokio_rustls::TlsAcceptor::from(server_config) + .accept(stream) + .await + .expect("TLS handshake should succeed"); + assert_eq!( + stream.get_ref().1.alpn_protocol(), + Some(b"http/1.1".as_slice()) + ); + assert_eq!( + stream + .get_ref() + .1 + .server_name() + .expect("client should send SNI"), + "localhost" + ); + let mut request = Vec::new(); + let mut buffer = [0_u8; 256]; + loop { + let read = tokio::io::AsyncReadExt::read(&mut stream, &mut buffer) + .await + .expect("HTTPS request should be readable"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let request = String::from_utf8(request).expect("request should be ASCII"); + assert!(request.starts_with("GET /resource?q=rust HTTP/1.1\r\n")); + assert!(request.contains(&format!("host: localhost:{}\r\n", address.port()))); + tokio::io::AsyncWriteExt::write_all( + &mut stream, + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ) + .await + .expect("HTTPS response should be writable"); + }); + let config = HttpConfig { + allowed_schemes: vec!["https".to_string()], + allowed_hosts: vec!["localhost".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + max_response_body_bytes: 2, + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("https://localhost:{}/resource?q=rust", address.port()) + .parse() + .expect("valid HTTPS URL"), + headers: Vec::new(), + body: None, + }; + let observer = ResponseReadObserver::default(); + let response = runtime + .block_on(execute_request_with_tls_config( + &config, + &request, + observer.clone(), + client_config, + )) + .expect("HTTPS request should complete"); + assert_eq!( + response.get(&Value::string("body")), + Some(&Value::bytes(b"ok".to_vec())) + ); + assert!(observer.max_raw_transport_read() > 0); + assert!(observer.max_raw_transport_read() <= 16_384 + 2_048 + 5); + runtime + .block_on(server) + .expect("TLS server should complete"); + } + + #[test] + fn accepted_tcp_with_stalled_tls_uses_the_connection_stage_deadline() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address"); + let server = std::thread::spawn(move || { + let (_socket, _) = listener.accept().expect("TCP client should connect"); + std::thread::sleep(Duration::from_millis(200)); + }); + let config = HttpConfig { + allowed_schemes: vec!["https".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + connect_timeout: Duration::from_millis(30), + request_timeout: Duration::from_secs(1), + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("https://{address}/") + .parse() + .expect("valid HTTPS URL"), + headers: Vec::new(), + body: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + let started = Instant::now(); + let error = runtime + .block_on(execute_request(&config, &request)) + .expect_err("stalled TLS must time out"); + assert!(error.to_string().contains("deadline exceeded")); + assert!(started.elapsed() < Duration::from_millis(150)); + server.join().expect("server should exit"); + } + + #[test] + fn request_deadline_caps_the_connection_stage_deadline() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener address"); + let server = std::thread::spawn(move || { + let (_socket, _) = listener.accept().expect("TCP client should connect"); + std::thread::sleep(Duration::from_millis(200)); + }); + let config = HttpConfig { + allowed_schemes: vec!["https".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + connect_timeout: Duration::from_secs(1), + request_timeout: Duration::from_millis(30), + ..HttpConfig::default() + }; + let request = HttpRequest { + method: hyper::Method::GET, + url: format!("https://{address}/") + .parse() + .expect("valid HTTPS URL"), + headers: Vec::new(), + body: None, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + let started = Instant::now(); + let error = runtime + .block_on(execute_request(&config, &request)) + .expect_err("request deadline must cap stalled TLS"); + assert!(error.to_string().contains("deadline exceeded")); + assert!(started.elapsed() < Duration::from_millis(150)); + server.join().expect("server should exit"); + } + + #[test] + fn dropping_host_future_aborts_connection_and_closes_peer_promptly() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("runtime should build"); + runtime.block_on(async { + let (client, mut server) = tokio::io::duplex(4096); + let (response_written, response_ready) = tokio::sync::oneshot::channel(); + let mut pending = pending_connection_test( + client, + "http://example.test/pending".parse().expect("valid URL"), + ); + let task = tokio::spawn(async move { + let mut request = Vec::new(); + let mut buffer = [0_u8; 256]; + loop { + let read = tokio::io::AsyncReadExt::read(&mut server, &mut buffer) + .await + .expect("request should be readable"); + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + tokio::io::AsyncWriteExt::write_all( + &mut server, + b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\na", + ) + .await + .expect("partial response should be writable"); + response_written + .send(()) + .expect("response readiness should be observed"); + let read = tokio::time::timeout( + Duration::from_millis(100), + tokio::io::AsyncReadExt::read(&mut server, &mut buffer), + ) + .await + .expect("peer EOF should be prompt") + .expect("peer EOF read should succeed"); + assert_eq!(read, 0); + }); + assert!( + futures_util::poll!(&mut pending.future).is_pending(), + "request should remain pending on the partial body" + ); + response_ready + .await + .expect("partial response should become ready"); + assert!( + futures_util::poll!(&mut pending.future).is_pending(), + "request should still await the remaining body" + ); + drop(pending); + task.await.expect("peer should observe EOF"); + }); + } + + #[test] + fn head_and_bodyless_statuses_ignore_declared_body_lengths() { + for (method, response, expected_status) in [ + ( + hyper::Method::HEAD, + b"HTTP/1.1 200 OK\r\nContent-Length: 999\r\n\r\n".as_slice(), + 200, + ), + ( + hyper::Method::GET, + b"HTTP/1.1 204 No Content\r\nContent-Length: 999\r\n\r\n".as_slice(), + 204, + ), + ( + hyper::Method::GET, + b"HTTP/1.1 304 Not Modified\r\nContent-Length: 999\r\n\r\n".as_slice(), + 304, + ), + ] { + let (result, observer) = + execute_fixture_response_fragments_for(method, vec![response], 1); + let response = result.expect("bodyless response should succeed"); + assert_eq!( + response.get(&Value::string("status")), + Some(&Value::Int(expected_status)) + ); + assert_eq!( + response.get(&Value::string("body")), + Some(&Value::bytes(Vec::new())) + ); + assert_eq!(observer.body_read_calls(), 0); + } + } + + #[test] + fn chunked_response_accepts_trailers_without_adding_them_to_the_body() { + let (result, _) = execute_fixture_response_fragments( + vec![ + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nTrailer: X-Checksum\r\n\r\n", + b"2\r\nok\r\n0\r\nX-Checksum: yes\r\n\r\n", + ], + 2, + ); + let response = result.expect("chunked response with trailers should succeed"); + assert_eq!( + response.get(&Value::string("body")), + Some(&Value::bytes(b"ok".to_vec())) + ); + } + + #[test] + fn truncated_content_length_propagates_a_body_or_connection_error() { + let (result, _) = execute_fixture_response( + b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nok", + 8, + ); + let error = result.expect_err("truncated response body must fail"); + let message = error.to_string(); + assert!( + message.contains("response read failed") || message.contains("connection failed"), + "unexpected error: {message}" + ); + } + + #[test] + fn oversized_response_head_is_rejected_by_the_hyper_buffer_bound() { + let oversized = format!( + "HTTP/1.1 200 OK\r\nX-Oversized: {}\r\nContent-Length: 0\r\n\r\n", + "a".repeat(70 * 1024) + ); + let response: &'static [u8] = Box::leak(oversized.into_bytes().into_boxed_slice()); + let (result, _) = execute_fixture_response(response, 1); + let error = result.expect_err("oversized response head must fail"); + let message = error.to_string(); + assert!( + message.contains("HTTP request failed") + || message.contains("connection failed before the response"), + "unexpected error: {message}" + ); + } + + #[test] + fn declared_oversized_body_is_rejected_before_body_transport_polling() { + let (result, observer) = execute_fixture_response( + b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nabcde", + 4, + ); + let error = result.expect_err("declared oversized body must fail"); + assert!(error.to_string().contains("response body exceeds limit")); + assert_eq!(observer.body_read_calls(), 0); + assert_eq!(observer.max_body_transport_read(), 0); + } + + #[test] + fn chunked_single_write_is_observed_only_through_remaining_plus_sentinel() { + let (result, observer) = execute_fixture_response( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n5\r\nabcde\r\n0\r\n\r\n", + 4, + ); + let error = result.expect_err("chunked limit plus one must fail"); + assert!(error.to_string().contains("response body exceeds limit")); + assert!(observer.body_read_calls() > 0); + assert!(observer.max_body_transport_read() <= 5); + assert!(observer.max_application_chunk() <= 5); + } + + #[test] + fn unknown_length_body_at_exact_limit_succeeds() { + let (result, observer) = + execute_fixture_response(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nabcd", 4); + let response = result.expect("exact-limit body should succeed"); + assert_eq!( + response.get(&Value::string("body")), + Some(&Value::bytes(b"abcd".to_vec())) + ); + assert!(observer.max_body_transport_read() <= 5); + assert!(observer.max_application_chunk() <= 4); + } + + #[test] + fn unknown_length_body_at_limit_plus_one_reads_only_the_sentinel() { + let (result, observer) = + execute_fixture_response(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nabcde", 4); + let error = result.expect_err("limit plus one body must fail"); + assert!(error.to_string().contains("response body exceeds limit")); + assert!(observer.max_body_transport_read() <= 5); + assert!(observer.max_application_chunk() <= 5); + } + + #[test] + fn empty_port_allowlist_rejects_explicit_and_default_ports() { + let config = HttpConfig { + allowed_schemes: vec!["https".to_string()], + allowed_hosts: vec!["example.com".to_string()], + ..HttpConfig::default() + }; + let explicit = "https://example.com:443/".parse().expect("valid URL"); + let default_port = "https://example.com/".parse().expect("valid URL"); + assert!(validate_url(&config, SchemeFamily::Http, &explicit).is_err()); + assert!(validate_url(&config, SchemeFamily::Http, &default_port).is_err()); + } + + #[test] + fn pinned_resolution_preserves_the_original_host_and_validated_address() { + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![8080], + allow_private_ips: true, + ..HttpConfig::default() + }; + let url = "http://127.0.0.1:8080/".parse().expect("valid pinned URL"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + let target = runtime + .block_on(super::policy::resolve_url( + &config, + SchemeFamily::Http, + &url, + )) + .expect("target should resolve under policy"); + + assert_eq!(target.host, "127.0.0.1"); + assert_eq!(target.address, "127.0.0.1:8080".parse().unwrap()); + } + + #[test] + fn special_use_networks_and_mixed_dns_answers_are_restricted() { + for address in [ + "0.1.2.3", + "100.64.0.1", + "192.0.0.8", + "192.0.2.1", + "192.31.196.1", + "192.52.193.1", + "192.88.99.1", + "192.175.48.1", + "198.18.0.1", + "198.51.100.1", + "203.0.113.1", + "240.0.0.1", + "100::1", + "2001::1", + "2001:db8::1", + "2002::1", + "2620:4f:8000::1", + "3fff::1", + "fc00::1", + ] { + assert!( + is_restricted_ip(address.parse().expect("valid IP")), + "{address} must be restricted" + ); + } + for address in ["8.8.8.8", "1.1.1.1", "2606:4700:4700::1111"] { + assert!( + !is_restricted_ip(address.parse().expect("valid IP")), + "{address} must remain globally routable" + ); + } + + let config = HttpConfig::default(); + let addresses = [ + "8.8.8.8:443".parse().expect("valid socket address"), + "100.64.0.1:443".parse().expect("valid socket address"), + ]; + assert!(validate_resolved_addresses(&config, &addresses).is_err()); + } + + #[test] + fn ipv4_mapped_ipv6_loopback_is_restricted() { + assert!(is_restricted_ip( + "::ffff:127.0.0.1".parse().expect("valid IP") + )); + } +} diff --git a/src/builtins/runtime/http/policy.rs b/src/builtins/runtime/http/policy.rs new file mode 100644 index 00000000..e132cb82 --- /dev/null +++ b/src/builtins/runtime/http/policy.rs @@ -0,0 +1,254 @@ +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +use super::config::HttpConfig; +use crate::vm::{VmError, VmResult}; + +/// URL scheme family admitted by a protocol adapter. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum SchemeFamily { + Http, +} + +impl SchemeFamily { + fn accepts(self, scheme: &str) -> bool { + match self { + Self::Http => matches!(scheme, "http" | "https"), + } + } +} + +#[derive(Clone, Debug)] +pub(super) struct ResolvedTarget { + pub(super) host: String, + pub(super) address: SocketAddr, +} + +/// Shared admission state for every connection-oriented HTTP adapter. +#[derive(Clone, Debug)] +pub(super) struct ConnectionAdmission { + max_in_flight: usize, + in_flight: Arc, +} + +impl ConnectionAdmission { + pub(super) fn new(max_in_flight: usize) -> Self { + Self { + max_in_flight, + in_flight: Arc::new(AtomicUsize::new(0)), + } + } + + pub(super) fn set_max_in_flight(&mut self, max_in_flight: usize) { + self.max_in_flight = max_in_flight; + } + + pub(super) fn max_in_flight(&self) -> usize { + self.max_in_flight + } + + pub(super) fn acquire(&self) -> VmResult { + let mut active = self.in_flight.load(Ordering::Acquire); + loop { + if active >= self.max_in_flight { + return Err(VmError::HostError(format!( + "HTTP in-flight request limit of {} was reached", + self.max_in_flight + ))); + } + match self.in_flight.compare_exchange_weak( + active, + active + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + return Ok(ConnectionPermit { + in_flight: Arc::clone(&self.in_flight), + }); + } + Err(observed) => active = observed, + } + } + } +} + +/// Releases one shared connection slot when its embedding-owned future retires. +pub(super) struct ConnectionPermit { + in_flight: Arc, +} + +impl Drop for ConnectionPermit { + fn drop(&mut self) { + self.in_flight.fetch_sub(1, Ordering::AcqRel); + } +} + +pub(super) fn validate_url_policy( + config: &HttpConfig, + family: SchemeFamily, + url: &url::Url, +) -> VmResult<(String, u16)> { + validate_url_structure(url)?; + let scheme = url.scheme().to_ascii_lowercase(); + if !family.accepts(&scheme) + || !config + .allowed_schemes + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(&scheme)) + { + return Err(VmError::HostError(format!( + "HTTP URL scheme '{scheme}' is not allowed", + ))); + } + let host = url + .host_str() + .expect("structurally validated HTTP URL must have a host"); + if !config + .allowed_hosts + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(host)) + { + return Err(VmError::HostError( + "HTTP target host is not allowed".to_string(), + )); + } + let port = url + .port_or_known_default() + .ok_or_else(|| VmError::HostError("HTTP URL has no known port".to_string()))?; + if !config.allowed_ports.contains(&port) { + return Err(VmError::HostError(format!( + "HTTP target port {port} is not allowed", + ))); + } + Ok((host.to_string(), port)) +} + +fn validate_url_structure(url: &url::Url) -> VmResult<()> { + if !url.username().is_empty() || url.password().is_some() { + return Err(VmError::HostError( + "HTTP URL userinfo is not allowed".to_string(), + )); + } + url.host_str() + .ok_or_else(|| VmError::HostError("HTTP URL has no host".to_string()))?; + Ok(()) +} + +pub(super) async fn resolve_url( + config: &HttpConfig, + family: SchemeFamily, + url: &url::Url, +) -> VmResult { + let (host, port) = validate_url_policy(config, family, url)?; + let addresses = if let Ok(host_ip) = host.parse::() { + vec![SocketAddr::new(host_ip, port)] + } else { + tokio::net::lookup_host((host.as_str(), port)) + .await + .map_err(|error| VmError::HostError(format!("HTTP host resolution failed: {error}")))? + .collect::>() + }; + validate_resolved_addresses(config, &addresses)?; + let address = addresses + .first() + .copied() + .ok_or_else(|| VmError::HostError("HTTP target resolves to a restricted IP".to_string()))?; + Ok(ResolvedTarget { host, address }) +} + +pub(super) fn validate_resolved_addresses( + config: &HttpConfig, + addresses: &[SocketAddr], +) -> VmResult<()> { + if addresses.is_empty() + || (!config.allow_private_ips + && addresses + .iter() + .any(|address| is_restricted_ip(address.ip()))) + { + return Err(VmError::HostError( + "HTTP target resolves to a restricted IP".to_string(), + )); + } + Ok(()) +} + +pub(super) fn is_restricted_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => { + let octets = ip.octets(); + matches!(octets[0], 0 | 10 | 127) + || (octets[0] == 100 && (64..=127).contains(&octets[1])) + || (octets[0] == 169 && octets[1] == 254) + || (octets[0] == 172 && (16..=31).contains(&octets[1])) + || (octets[0] == 192 + && matches!( + (octets[1], octets[2]), + (0, 0) | (0, 2) | (31, 196) | (52, 193) | (88, 99) | (168, _) | (175, 48) + )) + || (octets[0] == 198 + && ((18..=19).contains(&octets[1]) || (octets[1] == 51 && octets[2] == 100))) + || (octets[0] == 203 && octets[1] == 0 && octets[2] == 113) + || octets[0] >= 224 + } + IpAddr::V6(ip) => { + if let Some(mapped) = ip.to_ipv4_mapped() { + return is_restricted_ip(IpAddr::V4(mapped)); + } + let segments = ip.segments(); + let outside_global_unicast = segments[0] & 0xe000 != 0x2000; + let protocol_assignments = segments[0] == 0x2001 && segments[1] <= 0x01ff; + let documentation = (segments[0] == 0x2001 && segments[1] == 0x0db8) + || (segments[0] == 0x3fff && segments[1] & 0xf000 == 0); + let six_to_four = segments[0] == 0x2002; + let direct_delegation_as112 = + segments[0] == 0x2620 && segments[1] == 0x004f && segments[2] == 0x8000; + outside_global_unicast + || protocol_assignments + || documentation + || six_to_four + || direct_delegation_as112 + } + } +} + +pub(super) async fn with_deadline( + deadline: Instant, + future: impl std::future::Future>, +) -> VmResult { + tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), future) + .await + .map_err(|_| VmError::HostError("HTTP request deadline exceeded".to_string()))? +} + +pub(super) fn request_deadline(timeout: std::time::Duration) -> VmResult { + Instant::now().checked_add(timeout).ok_or_else(|| { + VmError::HostError("HTTP request_timeout cannot form a deadline".to_string()) + }) +} + +#[cfg(test)] +pub(super) fn validate_url( + config: &HttpConfig, + family: SchemeFamily, + url: &url::Url, +) -> VmResult> { + let (host, port) = validate_url_policy(config, family, url)?; + if config.allow_private_ips { + return Ok(None); + } + if let Ok(host_ip) = host.parse::() { + validate_resolved_addresses(config, &[SocketAddr::new(host_ip, port)])?; + return Ok(None); + } + use std::net::ToSocketAddrs; + let addresses = (host.as_str(), port) + .to_socket_addrs() + .map_err(|error| VmError::HostError(format!("HTTP host resolution failed: {error}")))? + .collect::>(); + validate_resolved_addresses(config, &addresses)?; + Ok(addresses.first().copied()) +} diff --git a/src/builtins/runtime/http/request.rs b/src/builtins/runtime/http/request.rs new file mode 100644 index 00000000..12287685 --- /dev/null +++ b/src/builtins/runtime/http/request.rs @@ -0,0 +1,960 @@ +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; +use std::task::{Context, Poll}; +use std::time::Instant; + +use futures_util::task::AtomicWaker; +use http_body_util::BodyExt; +use hyper::body::Body as _; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +use super::HttpRequestContext; +use super::config::HttpConfig; +use super::policy::{SchemeFamily, request_deadline, resolve_url, with_deadline}; +use crate::builtins::runtime::VmMap; +use crate::vm::{Value, VmError, VmResult}; + +#[derive(Clone, Default)] +pub(super) struct ResponseReadObserver { + inner: Arc, +} + +#[derive(Default)] +struct ResponseReadMetrics { + phase: AtomicU8, + transport_waker: AtomicWaker, + remaining_body_bytes: AtomicUsize, + body_read_calls: AtomicUsize, + max_body_transport_read: AtomicUsize, + max_raw_transport_read: AtomicUsize, + max_application_chunk: AtomicUsize, +} + +impl ResponseReadObserver { + fn mark_final_head(&self) { + self.inner.phase.store(1, Ordering::Release); + } + + pub(super) fn admit_body(&self, limit: usize) { + self.inner + .remaining_body_bytes + .store(limit, Ordering::Release); + self.inner.phase.store(2, Ordering::Release); + self.inner.transport_waker.wake(); + } + + fn body_is_admitted(&self) -> bool { + self.inner.phase.load(Ordering::Acquire) == 2 + } + + /// Discards the body admission state of a redirect response before the + /// next response is parsed. Redirect bodies are intentionally not exposed + /// to the caller, so carrying phase `1` into the next response would make + /// its body reader wait forever for an admission that can only happen after + /// `execute_request_until` returns. + fn discard_redirect_body(&self) { + self.inner.remaining_body_bytes.store(0, Ordering::Release); + self.inner.phase.store(0, Ordering::Release); + } + + fn register_transport_waker(&self, waker: &std::task::Waker) { + self.inner.transport_waker.register(waker); + } + + fn transport_read_limit(&self) -> usize { + if !self.body_is_admitted() { + 1 + } else { + self.inner + .remaining_body_bytes + .load(Ordering::Acquire) + .saturating_add(1) + } + } + + fn observe_transport_read(&self, bytes: usize) { + if self.body_is_admitted() { + self.inner.body_read_calls.fetch_add(1, Ordering::AcqRel); + self.inner + .max_body_transport_read + .fetch_max(bytes, Ordering::AcqRel); + } + } + + fn observe_raw_transport_read(&self, bytes: usize) { + self.inner + .max_raw_transport_read + .fetch_max(bytes, Ordering::AcqRel); + } + + pub(super) fn observe_application_chunk(&self, bytes: usize) { + self.inner + .max_application_chunk + .fetch_max(bytes, Ordering::AcqRel); + self.inner + .remaining_body_bytes + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + Some(remaining.saturating_sub(bytes)) + }) + .expect("response body remaining-byte update cannot fail"); + } + + #[cfg(test)] + pub(super) fn body_read_calls(&self) -> usize { + self.inner.body_read_calls.load(Ordering::Acquire) + } + + #[cfg(test)] + pub(super) fn max_body_transport_read(&self) -> usize { + self.inner.max_body_transport_read.load(Ordering::Acquire) + } + + #[cfg(test)] + pub(super) fn max_raw_transport_read(&self) -> usize { + self.inner.max_raw_transport_read.load(Ordering::Acquire) + } + + #[cfg(test)] + pub(super) fn max_application_chunk(&self) -> usize { + self.inner.max_application_chunk.load(Ordering::Acquire) + } +} + +// Rustls accepts a 16 KiB TLS fragment plus at most 2 KiB of protocol +// expansion and the five-byte record header. Bounding the adapter below TLS +// makes raw socket reads explicit. Rustls may retain one such record after the +// final HTTP head; ReadCapIo still exposes only remaining application bytes +// plus one overflow sentinel to Hyper. +const TLS_MAX_WIRE_READ: usize = 16_384 + 2_048 + 5; +const HTTP_MAX_HEAD_BYTES: usize = 64 * 1024; + +struct RawReadCapIo { + inner: T, + observer: ResponseReadObserver, +} + +impl RawReadCapIo { + fn new(inner: T, observer: ResponseReadObserver) -> Self { + Self { inner, observer } + } +} + +impl AsyncRead for RawReadCapIo { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + let before = buf.filled().len(); + let mut bounded = buf.take(TLS_MAX_WIRE_READ); + match Pin::new(&mut this.inner).poll_read(cx, &mut bounded) { + Poll::Ready(Ok(())) => { + let read = bounded.filled().len(); + let initialized = bounded.initialized().len(); + unsafe { + buf.assume_init(initialized); + buf.set_filled(before + read); + } + this.observer.observe_raw_transport_read(read); + Poll::Ready(Ok(())) + } + other => other, + } + } +} + +impl AsyncWrite for RawReadCapIo { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_shutdown(cx) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_write_vectored(cx, bufs) + } +} + +struct ReadCapIo { + inner: T, + observer: ResponseReadObserver, + header_suffix: [u8; 4], + header_bytes: usize, + header_complete: bool, + status_prefix: [u8; 12], + status_prefix_len: usize, +} + +impl ReadCapIo { + fn new(inner: T, observer: ResponseReadObserver) -> Self { + Self { + inner, + observer, + header_suffix: [0; 4], + header_bytes: 0, + header_complete: false, + status_prefix: [0; 12], + status_prefix_len: 0, + } + } + + fn observe_head_byte(&mut self, byte: u8) { + if self.status_prefix_len < self.status_prefix.len() { + self.status_prefix[self.status_prefix_len] = byte; + self.status_prefix_len += 1; + } + self.header_suffix.rotate_left(1); + self.header_suffix[3] = byte; + self.header_bytes = self.header_bytes.saturating_add(1); + if self.header_bytes < 4 || self.header_suffix != *b"\r\n\r\n" { + return; + } + + let status = std::str::from_utf8(&self.status_prefix[9..12]) + .ok() + .and_then(|digits| digits.parse::().ok()); + if matches!(status, Some(100..=199)) && status != Some(101) { + self.header_suffix = [0; 4]; + self.header_bytes = 0; + self.status_prefix = [0; 12]; + self.status_prefix_len = 0; + } else { + self.header_complete = true; + self.observer.mark_final_head(); + } + } +} + +impl AsyncRead for ReadCapIo { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + if this.header_complete && !this.observer.body_is_admitted() { + this.observer.register_transport_waker(cx.waker()); + if !this.observer.body_is_admitted() { + return Poll::Pending; + } + } + let before = buf.filled().len(); + let mut bounded = buf.take(this.observer.transport_read_limit()); + match Pin::new(&mut this.inner).poll_read(cx, &mut bounded) { + Poll::Ready(Ok(())) => { + let read = bounded.filled().len(); + let initialized = bounded.initialized().len(); + for byte in &bounded.filled()[..read] { + this.observe_head_byte(*byte); + } + unsafe { + buf.assume_init(initialized); + buf.set_filled(before + read); + } + this.observer.observe_transport_read(read); + Poll::Ready(Ok(())) + } + other => other, + } + } +} + +impl AsyncWrite for ReadCapIo { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_shutdown(cx) + } + + fn is_write_vectored(&self) -> bool { + self.inner.is_write_vectored() + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[std::io::IoSlice<'_>], + ) -> Poll> { + Pin::new(&mut self.get_mut().inner).poll_write_vectored(cx, bufs) + } +} + +#[derive(Clone)] +pub(super) struct HttpRequest { + pub(super) method: hyper::Method, + pub(super) url: url::Url, + pub(super) headers: Vec<(hyper::header::HeaderName, hyper::header::HeaderValue)>, + pub(super) body: Option>, +} + +pub(super) fn parse_request(map: &VmMap, config: &HttpConfig) -> VmResult { + let method = map_string(map, "method")?.to_ascii_uppercase(); + if !matches!( + method.as_str(), + "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" + ) { + return Err(VmError::HostError(format!( + "HTTP method '{method}' is not allowed" + ))); + } + let method = hyper::Method::from_bytes(method.as_bytes()) + .map_err(|_| VmError::HostError("invalid HTTP method".to_string()))?; + let url = map_string(map, "url")? + .parse::() + .map_err(|error| VmError::HostError(format!("invalid HTTP URL: {error}")))?; + + let body = match map.get(&Value::string("body")) { + None | Some(Value::Null) => None, + Some(Value::Bytes(bytes)) => { + if bytes.len() > config.max_request_body_bytes { + return Err(VmError::HostError( + "HTTP request body exceeds limit".to_string(), + )); + } + Some(bytes.as_ref().clone()) + } + Some(Value::String(text)) => { + if text.len() > config.max_request_body_bytes { + return Err(VmError::HostError( + "HTTP request body exceeds limit".to_string(), + )); + } + Some(text.as_bytes().to_vec()) + } + Some(_) => return Err(VmError::TypeMismatch("HTTP request body")), + }; + + let mut headers = Vec::new(); + if let Some(Value::Map(header_map)) = map.get(&Value::string("headers")) { + for (key, value) in header_map.iter() { + let Value::String(key) = key else { + return Err(VmError::TypeMismatch("HTTP header name")); + }; + let Value::String(value) = value else { + return Err(VmError::TypeMismatch("HTTP header value")); + }; + if matches!( + key.to_ascii_lowercase().as_str(), + "host" | "content-length" | "transfer-encoding" | "connection" + ) { + return Err(VmError::HostError(format!( + "HTTP header '{key}' is managed by the client", + ))); + } + let name = hyper::header::HeaderName::from_bytes(key.as_bytes()) + .map_err(|_| VmError::HostError(format!("invalid HTTP header name '{key}'")))?; + let value = hyper::header::HeaderValue::from_str(value).map_err(|_| { + VmError::HostError(format!("invalid HTTP header value for '{key}'")) + })?; + headers.push((name, value)); + } + } else if map.get(&Value::string("headers")).is_some() { + return Err(VmError::TypeMismatch("HTTP headers")); + } + + Ok(HttpRequest { + method, + url, + headers, + body, + }) +} + +fn map_string(map: &VmMap, key: &str) -> VmResult { + match map.get(&Value::string(key)) { + Some(Value::String(value)) => Ok(value.as_ref().clone()), + Some(_) => Err(VmError::TypeMismatch("HTTP request string field")), + None => Err(VmError::HostError(format!( + "missing HTTP request field '{key}'" + ))), + } +} + +pub(super) async fn perform_buffered_request( + context: HttpRequestContext, + request: VmMap, +) -> VmResult { + let request = parse_request(&request, &context.config)?; + let deadline = request_deadline(context.config.request_timeout)?; + with_deadline( + deadline, + execute_request_until( + &context.config, + &request, + ResponseReadObserver::default(), + deadline, + None, + ), + ) + .await +} + +#[cfg(test)] +pub(super) async fn execute_request(config: &HttpConfig, request: &HttpRequest) -> VmResult { + let deadline = request_deadline(config.request_timeout)?; + with_deadline( + deadline, + execute_request_until( + config, + request, + ResponseReadObserver::default(), + deadline, + None, + ), + ) + .await +} + +#[cfg(test)] +pub(super) async fn execute_request_with_observer( + config: &HttpConfig, + request: &HttpRequest, + observer: ResponseReadObserver, +) -> VmResult { + let deadline = request_deadline(config.request_timeout)?; + with_deadline( + deadline, + execute_request_until(config, request, observer, deadline, None), + ) + .await +} + +#[cfg(test)] +pub(super) async fn execute_request_with_tls_config( + config: &HttpConfig, + request: &HttpRequest, + observer: ResponseReadObserver, + tls_config: Arc, +) -> VmResult { + let deadline = request_deadline(config.request_timeout)?; + with_deadline( + deadline, + execute_request_until(config, request, observer, deadline, Some(tls_config)), + ) + .await +} + +async fn execute_request_until( + config: &HttpConfig, + request: &HttpRequest, + observer: ResponseReadObserver, + request_deadline: Instant, + tls_config: Option>, +) -> VmResult { + let mut method = request.method.clone(); + let mut url = request.url.clone(); + let mut body = request.body.clone(); + let mut headers = request.headers.clone(); + + for redirect_index in 0..=config.max_redirects { + let connect_deadline = request_deadline.min( + Instant::now() + .checked_add(config.connect_timeout) + .ok_or_else(|| { + VmError::HostError("HTTP connect_timeout cannot form a deadline".to_string()) + })?, + ); + let resolved = with_deadline( + connect_deadline, + resolve_url(config, SchemeFamily::Http, &url), + ) + .await?; + let origin = url.origin(); + let mut response = send_request( + &method, + &url, + &resolved, + &headers, + body.as_deref(), + ConnectionStage { + observer: observer.clone(), + deadline: connect_deadline, + tls_config: tls_config.clone(), + }, + ) + .await?; + if follows_location(response.response().status()) { + if redirect_index == config.max_redirects { + return Err(VmError::HostError( + "HTTP redirect limit exceeded".to_string(), + )); + } + let location = response + .response() + .headers() + .get(hyper::header::LOCATION) + .ok_or_else(|| VmError::HostError("HTTP redirect has no location".to_string()))? + .to_str() + .map_err(|_| VmError::HostError("HTTP redirect location is invalid".to_string()))? + .to_string(); + let next_url = url + .join(&location) + .map_err(|error| VmError::HostError(format!("invalid HTTP redirect: {error}")))?; + if next_url.origin() != origin { + headers.retain(|(name, _)| { + name != hyper::header::AUTHORIZATION && name != hyper::header::COOKIE + }); + } + rewrite_redirect_request(response.response().status(), &mut method, &mut body); + drop(response); + observer.discard_redirect_body(); + url = next_url; + continue; + } + + let status = response.response().status(); + let has_body = response_has_body(&method, status); + if has_body { + reject_declared_oversize(response.response(), config.max_response_body_bytes)?; + } + let response_headers = response_header_entries(response.response().headers()); + if !has_body { + return Ok(response_map(status, response_headers, Vec::new(), &url)); + } + observer.admit_body(config.max_response_body_bytes); + let mut bytes = Vec::with_capacity( + response + .response() + .body() + .size_hint() + .exact() + .and_then(|length| usize::try_from(length).ok()) + .unwrap_or(0) + .min(config.max_response_body_bytes), + ); + while let Some(frame) = response.next_frame().await? { + let Ok(chunk) = frame.into_data() else { + continue; + }; + observer.observe_application_chunk(chunk.len()); + if bytes.len().saturating_add(chunk.len()) > config.max_response_body_bytes { + return Err(response_body_limit_error()); + } + bytes.extend_from_slice(&chunk); + } + return Ok(response_map(status, response_headers, bytes, &url)); + } + + Err(VmError::HostError( + "HTTP redirect processing failed".to_string(), + )) +} + +type BoxConnection = + Pin> + Send + 'static>>; + +pub(super) struct OwnedResponse { + connection: Option, + response: hyper::Response, +} + +impl OwnedResponse { + pub(super) fn response(&self) -> &hyper::Response { + &self.response + } + + pub(super) async fn next_frame( + &mut self, + ) -> VmResult>> { + enum Progress { + Frame(Option, hyper::Error>>), + Connection(Result<(), hyper::Error>), + } + + loop { + let Some(connection) = self.connection.as_mut() else { + return self + .response + .body_mut() + .frame() + .await + .transpose() + .map_err(|error| { + VmError::HostError(format!("HTTP response read failed: {error}")) + }); + }; + let progress = tokio::select! { + biased; + frame = self.response.body_mut().frame() => Progress::Frame(frame), + result = connection.as_mut() => Progress::Connection(result), + }; + match progress { + Progress::Frame(frame) => { + return frame.transpose().map_err(|error| { + VmError::HostError(format!("HTTP response read failed: {error}")) + }); + } + Progress::Connection(Ok(())) => self.connection = None, + Progress::Connection(Err(error)) => { + return Err(VmError::HostError(format!( + "HTTP connection failed: {error}" + ))); + } + } + } + } +} + +fn response_has_body(method: &hyper::Method, status: hyper::StatusCode) -> bool { + *method != hyper::Method::HEAD + && !status.is_informational() + && status != hyper::StatusCode::NO_CONTENT + && status != hyper::StatusCode::NOT_MODIFIED +} + +fn follows_location(status: hyper::StatusCode) -> bool { + matches!( + status, + hyper::StatusCode::MOVED_PERMANENTLY + | hyper::StatusCode::FOUND + | hyper::StatusCode::SEE_OTHER + | hyper::StatusCode::TEMPORARY_REDIRECT + | hyper::StatusCode::PERMANENT_REDIRECT + ) +} + +fn rewrite_redirect_request( + status: hyper::StatusCode, + method: &mut hyper::Method, + body: &mut Option>, +) { + let rewrite_to_get = match status { + hyper::StatusCode::SEE_OTHER => { + *method != hyper::Method::GET && *method != hyper::Method::HEAD + } + hyper::StatusCode::MOVED_PERMANENTLY | hyper::StatusCode::FOUND => { + *method == hyper::Method::POST + } + _ => false, + }; + if rewrite_to_get { + *method = hyper::Method::GET; + *body = None; + } +} + +pub(super) fn response_header_entries(headers: &hyper::HeaderMap) -> Vec<(Value, Value)> { + headers + .iter() + .map(|(name, value)| { + let value = value + .to_str() + .map(Value::string) + .unwrap_or_else(|_| Value::bytes(value.as_bytes().to_vec())); + (Value::string(name.as_str()), value) + }) + .collect() +} + +pub(super) async fn open_stream_response( + config: &HttpConfig, + request: &HttpRequest, + observer: ResponseReadObserver, + deadline: Option, +) -> VmResult<(OwnedResponse, url::Url)> { + let mut method = request.method.clone(); + let mut url = request.url.clone(); + let mut body = request.body.clone(); + let mut headers = request.headers.clone(); + for redirect_index in 0..=config.max_redirects { + let mut connect_deadline = Instant::now() + .checked_add(config.connect_timeout) + .ok_or_else(|| { + VmError::HostError("HTTP connect_timeout cannot form a deadline".to_string()) + })?; + if let Some(deadline) = deadline { + connect_deadline = connect_deadline.min(deadline); + } + let resolved = with_deadline( + connect_deadline, + resolve_url(config, SchemeFamily::Http, &url), + ) + .await?; + let origin = url.origin(); + let response = send_request( + &method, + &url, + &resolved, + &headers, + body.as_deref(), + ConnectionStage { + observer: observer.clone(), + deadline: connect_deadline, + tls_config: None, + }, + ) + .await?; + if follows_location(response.response().status()) { + if redirect_index == config.max_redirects { + return Err(VmError::HostError( + "HTTP redirect limit exceeded".to_string(), + )); + } + let location = response + .response() + .headers() + .get(hyper::header::LOCATION) + .ok_or_else(|| VmError::HostError("HTTP redirect has no location".to_string()))? + .to_str() + .map_err(|_| VmError::HostError("HTTP redirect location is invalid".to_string()))?; + let next_url = url + .join(location) + .map_err(|error| VmError::HostError(format!("invalid HTTP redirect: {error}")))?; + super::policy::validate_url_policy(config, SchemeFamily::Http, &next_url)?; + if next_url.origin() != origin { + headers.retain(|(name, _)| { + name != hyper::header::AUTHORIZATION && name != hyper::header::COOKIE + }); + } + rewrite_redirect_request(response.response().status(), &mut method, &mut body); + drop(response); + observer.discard_redirect_body(); + url = next_url; + continue; + } + return Ok((response, url)); + } + Err(VmError::HostError( + "HTTP redirect processing failed".to_string(), + )) +} + +fn response_map( + status: hyper::StatusCode, + headers: Vec<(Value, Value)>, + body: Vec, + url: &url::Url, +) -> VmMap { + VmMap::from_entries(vec![ + ( + Value::string("status"), + Value::Int(i64::from(status.as_u16())), + ), + ( + Value::string("headers"), + Value::Map(std::sync::Arc::new(VmMap::from_entries(headers))), + ), + (Value::string("body"), Value::bytes(body)), + (Value::string("url"), Value::string(url.as_str())), + ]) +} + +fn response_body_limit_error() -> VmError { + VmError::HostError("HTTP response body exceeds limit".to_string()) +} + +fn reject_declared_oversize( + response: &hyper::Response, + limit: usize, +) -> VmResult<()> { + let Some(value) = response.headers().get(hyper::header::CONTENT_LENGTH) else { + return Ok(()); + }; + let length = value + .to_str() + .ok() + .and_then(|text| text.parse::().ok()) + .ok_or_else(|| VmError::HostError("HTTP response Content-Length is invalid".to_string()))?; + if length > limit as u64 { + return Err(response_body_limit_error()); + } + Ok(()) +} + +struct ConnectionStage { + observer: ResponseReadObserver, + deadline: Instant, + tls_config: Option>, +} + +async fn send_request( + method: &hyper::Method, + url: &url::Url, + resolved: &super::policy::ResolvedTarget, + headers: &[(hyper::header::HeaderName, hyper::header::HeaderValue)], + body: Option<&[u8]>, + stage: ConnectionStage, +) -> VmResult { + let ConnectionStage { + observer, + deadline: connect_deadline, + tls_config, + } = stage; + let stream = with_deadline(connect_deadline, async { + tokio::net::TcpStream::connect(resolved.address) + .await + .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}"))) + }) + .await?; + stream + .set_nodelay(true) + .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}")))?; + + let raw = RawReadCapIo::new(stream, observer.clone()); + if url.scheme() == "https" { + let mut tls_config = tls_config.map_or_else( + || { + let mut roots = rustls::RootCertStore::empty(); + roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth() + }, + Arc::unwrap_or_clone, + ); + tls_config.alpn_protocols = vec![b"http/1.1".to_vec()]; + let server_name = rustls::pki_types::ServerName::try_from(resolved.host.clone()) + .map_err(|_| VmError::HostError("HTTP TLS server name is invalid".to_string()))?; + let stream = with_deadline(connect_deadline, async { + tokio_rustls::TlsConnector::from(Arc::new(tls_config)) + .connect(server_name, raw) + .await + .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}"))) + }) + .await?; + send_over_io(method, url, headers, body, ReadCapIo::new(stream, observer)).await + } else { + send_over_io(method, url, headers, body, ReadCapIo::new(raw, observer)).await + } +} + +#[cfg(test)] +pub(super) struct PendingConnectionTest { + pub(super) future: Pin>>>, +} + +#[cfg(test)] +pub(super) fn pending_connection_test( + io: tokio::io::DuplexStream, + url: url::Url, +) -> PendingConnectionTest { + let request = HttpRequest { + method: hyper::Method::GET, + url, + headers: Vec::new(), + body: None, + }; + let observer = ResponseReadObserver::default(); + PendingConnectionTest { + future: Box::pin(async move { + let mut response = send_over_io( + &request.method, + &request.url, + &request.headers, + None, + ReadCapIo::new(RawReadCapIo::new(io, observer.clone()), observer.clone()), + ) + .await?; + observer.admit_body(1024); + while response.next_frame().await?.is_some() {} + Ok(VmMap::default()) + }), + } +} + +async fn send_over_io( + method: &hyper::Method, + url: &url::Url, + headers: &[(hyper::header::HeaderName, hyper::header::HeaderValue)], + body: Option<&[u8]>, + io: ReadCapIo, +) -> VmResult +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let mut connection_builder = hyper::client::conn::http1::Builder::new(); + connection_builder + .read_buf_exact_size(Some(8 * 1024)) + .max_buf_size(HTTP_MAX_HEAD_BYTES) + .max_headers(100); + let (mut sender, connection) = connection_builder + .handshake(hyper_util::rt::TokioIo::new(io)) + .await + .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}")))?; + + let path_and_query = match url.query() { + Some(query) => format!("{}?{query}", url.path()), + None => url.path().to_string(), + }; + let mut builder = hyper::Request::builder() + .method(method.clone()) + .uri(path_and_query) + .header( + hyper::header::HOST, + &url[url::Position::BeforeHost..url::Position::AfterPort], + ); + for (name, value) in headers { + builder = builder.header(name, value); + } + let request_body = http_body_util::Full::new(hyper::body::Bytes::copy_from_slice( + body.unwrap_or_default(), + )); + let request = builder + .body(request_body) + .map_err(|error| VmError::HostError(format!("HTTP request setup failed: {error}")))?; + let mut connection: BoxConnection = Box::pin(connection); + let (response, connection) = { + let response = sender.send_request(request); + tokio::pin!(response); + tokio::select! { + biased; + response = &mut response => ( + response.map_err(|error| { + VmError::HostError(format!("HTTP request failed: {error}")) + })?, + Some(connection), + ), + connection_result = connection.as_mut() => { + let response_result = response.await; + let response = match (connection_result, response_result) { + (_, Ok(response)) => response, + (Ok(()), Err(error)) => { + return Err(VmError::HostError(format!( + "HTTP request failed: {error}" + ))); + } + (Err(connection_error), Err(request_error)) => { + return Err(VmError::HostError(format!( + "HTTP connection failed before the response: {connection_error}; request failed: {request_error}" + ))); + } + }; + (response, None) + } + } + }; + Ok(OwnedResponse { + connection, + response, + }) +} diff --git a/src/builtins/runtime/http/sse.rs b/src/builtins/runtime/http/sse.rs new file mode 100644 index 00000000..18d75748 --- /dev/null +++ b/src/builtins/runtime/http/sse.rs @@ -0,0 +1,899 @@ +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +use pd_host_function::pd_host_function; + +use super::request::{ + HttpRequest, OwnedResponse, ResponseReadObserver, open_stream_response, parse_request, + response_header_entries, +}; +use super::{HttpRequestContext, policy}; +use crate::builtins::runtime::typed::VmMapHandle; +use crate::builtins::runtime::{HostCallResult, VmCallable, VmMap}; +use crate::vm::{ + CallOutcome, HostStreamAction, HostStreamDriver, HostStreamPoll, Value, Vm, VmError, VmResult, +}; + +#[derive(Debug, PartialEq, Eq)] +struct SseEvent { + event: Option, + data: String, + id: Option, + retry_ms: Option, +} + +/// Incremental EventSource parser. `max_total_bytes` counts raw response-body +/// octets, including a BOM and line terminators. `max_item_bytes` counts the +/// UTF-8 bytes retained in data (including inserted joins), event, and id. +struct SseParser { + max_line_bytes: usize, + max_item_bytes: usize, + max_total_bytes: usize, + total_bytes: usize, + prefix: Vec, + bom_decided: bool, + line: Vec, + after_cr: bool, + data: String, + has_data: bool, + event: Option, + id: Option, + retry_ms: Option, + finished: bool, +} + +impl SseParser { + fn new(max_line_bytes: usize, max_item_bytes: usize, max_total_bytes: usize) -> Self { + Self { + max_line_bytes, + max_item_bytes, + max_total_bytes, + total_bytes: 0, + prefix: Vec::with_capacity(3), + bom_decided: false, + line: Vec::with_capacity(max_line_bytes.min(1024)), + after_cr: false, + data: String::new(), + has_data: false, + event: None, + id: None, + retry_ms: None, + finished: false, + } + } + + #[cfg(test)] + fn push(&mut self, bytes: &[u8]) -> VmResult> { + self.admit_chunk(bytes.len())?; + let mut events = Vec::new(); + let mut offset = 0; + while offset < bytes.len() { + let (consumed, event) = self.push_until_event(&bytes[offset..])?; + offset += consumed; + if let Some(event) = event { + events.push(event); + } + } + Ok(events) + } + + fn admit_chunk(&mut self, bytes: usize) -> VmResult<()> { + self.total_bytes = self + .total_bytes + .checked_add(bytes) + .filter(|total| *total <= self.max_total_bytes) + .ok_or_else(|| VmError::HostError("SSE stream exceeds total byte limit".to_string()))?; + Ok(()) + } + + fn push_until_event(&mut self, bytes: &[u8]) -> VmResult<(usize, Option)> { + if self.finished { + return Err(VmError::HostError( + "SSE parser received bytes after EOF".to_string(), + )); + } + let mut consumed = 0; + while consumed < bytes.len() { + let byte = bytes[consumed]; + consumed += 1; + if !self.bom_decided { + self.prefix.push(byte); + if self.prefix == b"\xef\xbb\xbf" { + self.prefix.clear(); + self.bom_decided = true; + continue; + } + if b"\xef\xbb\xbf".starts_with(&self.prefix) { + continue; + } + let prefix = std::mem::take(&mut self.prefix); + self.bom_decided = true; + for byte in prefix { + if let Some(event) = self.process_byte(byte)? { + return Ok((consumed, Some(event))); + } + } + continue; + } + if let Some(event) = self.process_byte(byte)? { + return Ok((consumed, Some(event))); + } + } + Ok((consumed, None)) + } + + fn finish(&mut self) -> VmResult> { + if self.finished { + return Ok(Vec::new()); + } + self.finished = true; + let mut events = Vec::new(); + if !self.prefix.is_empty() { + let prefix = std::mem::take(&mut self.prefix); + for byte in prefix { + if let Some(event) = self.process_byte(byte)? { + events.push(event); + } + } + } + if !self.line.is_empty() + && let Some(event) = self.process_line()? + { + events.push(event); + } + // EventSource dispatches only on a blank line. EOF discards a partial + // event, including a final unterminated data line. + self.data.clear(); + self.has_data = false; + self.event = None; + Ok(events) + } + + fn process_byte(&mut self, byte: u8) -> VmResult> { + if self.after_cr { + self.after_cr = false; + if byte == b'\n' { + return Ok(None); + } + } + match byte { + b'\r' => { + let event = self.process_line()?; + self.after_cr = true; + Ok(event) + } + b'\n' => self.process_line(), + _ => { + if self.line.len() == self.max_line_bytes { + return Err(VmError::HostError( + "SSE line exceeds byte limit".to_string(), + )); + } + self.line.push(byte); + Ok(None) + } + } + } + + fn process_line(&mut self) -> VmResult> { + let bytes = std::mem::take(&mut self.line); + let line = std::str::from_utf8(&bytes) + .map_err(|_| VmError::HostError("SSE stream contains malformed UTF-8".to_string()))?; + if line.is_empty() { + if self.data_seen() { + return Ok(Some(self.dispatch_event())); + } + // The WHATWG dispatch algorithm clears both data and event type + // buffers even when empty data causes dispatch to return early. + self.event = None; + return Ok(None); + } + if line.starts_with(':') { + return Ok(None); + } + let (field, mut value) = line.split_once(':').unwrap_or((line, "")); + if let Some(rest) = value.strip_prefix(' ') { + value = rest; + } + match field { + "data" => { + let added = value.len() + usize::from(self.has_data); + self.ensure_item_growth(added, self.event.as_deref(), self.id.as_deref())?; + if self.has_data { + self.data.push('\n'); + } + self.data.push_str(value); + self.has_data = true; + } + "event" => { + self.ensure_item_size(self.data.len(), Some(value), self.id.as_deref())?; + self.event = Some(value.to_string()); + } + "id" if !value.contains('\0') => { + self.ensure_item_size(self.data.len(), self.event.as_deref(), Some(value))?; + self.id = Some(value.to_string()); + } + "retry" if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) => { + if let Ok(retry) = value.parse::() { + self.retry_ms = Some(retry); + } + } + _ => {} + } + Ok(None) + } + + fn data_seen(&self) -> bool { + self.has_data + } + + fn ensure_item_growth( + &self, + added: usize, + event: Option<&str>, + id: Option<&str>, + ) -> VmResult<()> { + let data = self + .data + .len() + .checked_add(added) + .ok_or_else(item_limit_error)?; + self.ensure_item_size(data, event, id) + } + + fn ensure_item_size( + &self, + data_bytes: usize, + event: Option<&str>, + id: Option<&str>, + ) -> VmResult<()> { + let size = data_bytes + .checked_add(event.map_or(0, str::len)) + .and_then(|size| size.checked_add(id.map_or(0, str::len))) + .ok_or_else(item_limit_error)?; + if size > self.max_item_bytes { + return Err(item_limit_error()); + } + Ok(()) + } + + fn dispatch_event(&mut self) -> SseEvent { + let data = std::mem::take(&mut self.data); + self.has_data = false; + SseEvent { + event: self.event.take(), + data, + id: self.id.clone(), + retry_ms: self.retry_ms, + } + } +} + +fn item_limit_error() -> VmError { + VmError::HostError("SSE item exceeds byte limit".to_string()) +} + +type OpenFuture = Pin> + Send>>; +type FrameFuture = Pin< + Box< + dyn Future< + Output = ( + OwnedResponse, + VmResult>>, + ), + > + Send, + >, +>; + +enum DriverState { + Opening { + future: OpenFuture, + idle_deadline: Instant, + timeout: Option>>, + }, + Reading { + future: FrameFuture, + idle_deadline: Instant, + timeout: Pin>, + }, + Ready(OwnedResponse), + Closed, +} + +struct SseDriver { + state: DriverState, + parser: SseParser, + chunk: Option, + chunk_offset: usize, + eof_pending: bool, + config: super::HttpConfig, + observer: ResponseReadObserver, + permit: Option, + deadline: Instant, + status: Option, + headers: Option>, + url: Option, + items: i64, + bytes_received: i64, +} + +impl Drop for SseDriver { + fn drop(&mut self) { + self.retire(); + } +} + +impl SseDriver { + fn new(context: HttpRequestContext, request: HttpRequest, deadline: Instant) -> Self { + let super::HttpRequestContext { config, _permit } = context; + let observer = ResponseReadObserver::default(); + let open_config = config.clone(); + let open_observer = observer.clone(); + let future = Box::pin(async move { + open_stream_response(&open_config, &request, open_observer, Some(deadline)).await + }); + let idle_deadline = Instant::now() + .checked_add(config.stream_idle_timeout) + .expect("validated idle timeout"); + Self { + state: DriverState::Opening { + future, + idle_deadline, + timeout: None, + }, + parser: SseParser::new( + config.max_sse_line_bytes, + config.max_stream_item_bytes, + config.max_stream_total_bytes, + ), + chunk: None, + chunk_offset: 0, + eof_pending: false, + config, + observer, + permit: Some(_permit), + deadline, + status: None, + headers: None, + url: None, + items: 0, + bytes_received: 0, + } + } + + fn validate_response(&mut self, response: &OwnedResponse, url: url::Url) -> VmResult { + let status = response.response().status(); + if !status.is_success() { + return Err(VmError::HostError(format!( + "SSE response status {} is not successful", + status.as_u16() + ))); + } + let content_type = response + .response() + .headers() + .get(hyper::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .map(str::trim) + .filter(|value| value.eq_ignore_ascii_case("text/event-stream")) + .ok_or_else(|| { + VmError::HostError( + "SSE response Content-Type must be text/event-stream".to_string(), + ) + })?; + debug_assert!(content_type.eq_ignore_ascii_case("text/event-stream")); + let headers = std::sync::Arc::new(VmMap::from_entries(response_header_entries( + response.response().headers(), + ))); + self.status = Some(status); + self.headers = Some(std::sync::Arc::clone(&headers)); + self.url = Some(url.clone()); + self.observer.admit_body(self.config.max_stream_total_bytes); + Ok(map_value(vec![ + ("kind", Value::string("open")), + ("status", Value::Int(i64::from(status.as_u16()))), + ("headers", Value::Map(headers)), + ("url", Value::string(url.as_str())), + ])) + } + + fn event_value(event: SseEvent) -> Value { + map_value(vec![ + ("kind", Value::string("event")), + ("event", event.event.map_or(Value::Null, Value::string)), + ("data", Value::string(event.data)), + ("id", event.id.map_or(Value::Null, Value::string)), + ("retry_ms", event.retry_ms.map_or(Value::Null, Value::Int)), + ]) + } + + fn summary(&self, outcome: &str) -> Value { + map_value(vec![ + ("outcome", Value::string(outcome)), + ( + "status", + Value::Int(i64::from( + self.status.expect("summary requires open status").as_u16(), + )), + ), + ( + "headers", + Value::Map( + self.headers + .as_ref() + .expect("summary requires headers") + .clone(), + ), + ), + ( + "url", + Value::string(self.url.as_ref().expect("summary requires URL").as_str()), + ), + ("items", Value::Int(self.items)), + ("bytes_received", Value::Int(self.bytes_received)), + ("bytes_sent", Value::Int(0)), + ]) + } + + fn retire(&mut self) { + self.state = DriverState::Closed; + self.chunk = None; + self.eof_pending = false; + self.permit.take(); + } + + fn ensure_before_deadline(&mut self) -> VmResult<()> { + if Instant::now() >= self.deadline { + self.retire(); + return Err(VmError::HostError( + "SSE total deadline exceeded".to_string(), + )); + } + Ok(()) + } +} + +impl HostStreamDriver for SseDriver { + fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll> { + loop { + if let Err(error) = self.ensure_before_deadline() { + return Poll::Ready(Err(error)); + } + if let Some(chunk) = self.chunk.as_ref() { + let (consumed, event) = + self.parser.push_until_event(&chunk[self.chunk_offset..])?; + self.chunk_offset += consumed; + if self.chunk_offset == chunk.len() { + self.chunk = None; + self.chunk_offset = 0; + } + if let Some(event) = event { + return Poll::Ready(Ok(HostStreamPoll::Item(Self::event_value(event)))); + } + } + if self.eof_pending { + // `finish` only validates and cleans up: it can surface a + // partial BOM/UTF-8 or line-limit error, but it can never + // dispatch an event because EventSource dispatch requires a + // blank line and EOF discards a partial final event. + self.parser.finish()?; + self.eof_pending = false; + self.state = DriverState::Closed; + return Poll::Ready(Ok(HostStreamPoll::Item(map_value(vec![( + "kind", + Value::string("end"), + )])))); + } + match &mut self.state { + DriverState::Opening { + future, + idle_deadline, + timeout, + } => { + let open_deadline = self.deadline.min(*idle_deadline); + let timeout = timeout.get_or_insert_with(|| { + Box::pin(tokio::time::sleep_until(tokio::time::Instant::from_std( + open_deadline, + ))) + }); + if timeout.as_mut().poll(cx).is_ready() { + let total_expired = self.deadline <= *idle_deadline; + self.retire(); + return Poll::Ready(Err(VmError::HostError( + if total_expired { + "SSE total deadline exceeded" + } else { + "SSE stream idle timeout while opening response" + } + .to_string(), + ))); + } + match future.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(error)) => { + self.retire(); + return Poll::Ready(Err(error)); + } + Poll::Ready(Ok((response, url))) => { + let open = match self.validate_response(&response, url) { + Ok(open) => open, + Err(error) => { + self.retire(); + return Poll::Ready(Err(error)); + } + }; + self.state = DriverState::Ready(response); + return Poll::Ready(Ok(HostStreamPoll::Item(open))); + } + } + } + DriverState::Ready(_) => { + let DriverState::Ready(mut response) = + std::mem::replace(&mut self.state, DriverState::Closed) + else { + unreachable!() + }; + let idle_deadline = Instant::now() + .checked_add(self.config.stream_idle_timeout) + .expect("validated idle timeout"); + let deadline = self.deadline.min(idle_deadline); + self.state = DriverState::Reading { + future: Box::pin(async move { + let frame = response.next_frame().await; + (response, frame) + }), + idle_deadline, + timeout: Box::pin(tokio::time::sleep_until( + tokio::time::Instant::from_std(deadline), + )), + }; + } + DriverState::Reading { + future, + idle_deadline, + timeout, + } => { + if timeout.as_mut().poll(cx).is_ready() { + let total_expired = self.deadline <= *idle_deadline; + self.retire(); + return Poll::Ready(Err(VmError::HostError( + if total_expired { + "SSE total deadline exceeded" + } else { + "SSE stream idle timeout" + } + .to_string(), + ))); + } + match future.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready((response, Err(error))) => { + drop(response); + self.retire(); + return Poll::Ready(Err(error)); + } + Poll::Ready((response, Ok(Some(frame)))) => { + self.state = DriverState::Ready(response); + if let Ok(data) = frame.into_data() { + self.parser.admit_chunk(data.len())?; + self.observer.observe_application_chunk(data.len()); + self.bytes_received = self + .bytes_received + .checked_add(i64::try_from(data.len()).map_err(|_| { + VmError::HostError( + "SSE byte count exceeds script int".into(), + ) + })?) + .ok_or_else(|| { + VmError::HostError( + "SSE byte count exceeds script int".into(), + ) + })?; + self.chunk = Some(data); + self.chunk_offset = 0; + } + } + Poll::Ready((response, Ok(None))) => { + drop(response); + self.eof_pending = true; + } + } + } + DriverState::Closed => { + self.permit.take(); + return Poll::Ready(Ok(HostStreamPoll::Complete(self.summary("eof")))); + } + } + } + } + + fn apply_action(&mut self, action: Value) -> VmResult { + self.ensure_before_deadline()?; + let Value::Map(action) = action else { + self.retire(); + return Err(VmError::HostError( + "SSE callback action must be a map".to_string(), + )); + }; + let Some(Value::String(action)) = action.get(&Value::string("action")) else { + self.retire(); + return Err(VmError::HostError( + "SSE callback action must contain string 'action'".to_string(), + )); + }; + self.items = self + .items + .checked_add(1) + .ok_or_else(|| VmError::HostError("SSE item count exceeds script int".to_string()))?; + match action.as_str() { + "continue" => Ok(HostStreamAction::Continue), + "stop" => { + let summary = self.summary("stopped"); + self.retire(); + Ok(HostStreamAction::Complete(summary)) + } + other => { + let error = VmError::HostError(format!("invalid SSE callback action '{other}'")); + self.retire(); + Err(error) + } + } + } +} + +fn map_value(entries: Vec<(&'static str, Value)>) -> Value { + Value::Map(std::sync::Arc::new(VmMap::from_entries( + entries + .into_iter() + .map(|(key, value)| (Value::string(key), value)) + .collect(), + ))) +} + +fn parse_stream_timeout(request: &VmMap) -> VmResult> { + let Some(value) = request.get(&Value::string("timeout_ms")) else { + return Ok(None); + }; + let Value::Int(milliseconds) = value else { + return Err(VmError::TypeMismatch("SSE timeout_ms")); + }; + let milliseconds = u64::try_from(*milliseconds) + .ok() + .filter(|milliseconds| *milliseconds > 0) + .ok_or_else(|| VmError::HostError("SSE timeout_ms must be positive".to_string()))?; + Ok(Some(Duration::from_millis(milliseconds))) +} + +/// Streams one bounded SSE item into one script callback at a time. +#[pd_host_function(name = "http::client::sse")] +pub(super) fn builtin_http_client_sse_impl( + vm: &mut Vm, + request: VmMapHandle, + on_event: VmCallable VmMap>, +) -> VmResult> { + let callback = on_event.into_value(); + vm.validate_stream_callback_value(&callback)?; + let script_timeout = parse_stream_timeout(request.as_ref())?; + let (context, deadline) = HttpRequestContext::capture_stream(vm, script_timeout, "SSE")?; + let mut request = parse_request(request.as_ref(), &context.config)?; + policy::validate_url_policy(&context.config, policy::SchemeFamily::Http, &request.url)?; + if request.method != hyper::Method::GET && request.method != hyper::Method::POST { + return Err(VmError::HostError( + "SSE requests require GET or POST".to_string(), + )); + } + if !request + .headers + .iter() + .any(|(name, _)| name == hyper::header::ACCEPT) + { + request.headers.push(( + hyper::header::ACCEPT, + hyper::header::HeaderValue::from_static("text/event-stream"), + )); + } + match vm.submit_callable_stream(callback, SseDriver::new(context, request, deadline))? { + CallOutcome::Pending(op_id) => Ok(HostCallResult::Pending(op_id)), + _ => Err(VmError::InvalidFrameState( + "callable stream admission returned a non-pending outcome", + )), + } +} + +#[cfg(test)] +mod tests { + use super::{SseEvent, SseParser}; + + fn event(data: &str, event: Option<&str>, id: Option<&str>, retry_ms: Option) -> SseEvent { + SseEvent { + event: event.map(str::to_string), + data: data.to_string(), + id: id.map(str::to_string), + retry_ms, + } + } + + fn parse_fragments( + fragments: &[&[u8]], + line: usize, + item: usize, + total: usize, + ) -> Result, String> { + let mut parser = SseParser::new(line, item, total); + let mut events = Vec::new(); + for fragment in fragments { + events.extend(parser.push(fragment).map_err(|error| error.to_string())?); + } + events.extend(parser.finish().map_err(|error| error.to_string())?); + Ok(events) + } + + #[test] + fn parser_accepts_fragmented_bom_utf8_and_every_line_ending() { + let fragments: &[&[u8]] = &[ + b"\xef", + b"\xbb\xbfdata: h\xc3", + b"\xa9\r", + b"data: two\n", + b"event:first\r\nevent: final\r", + b"id: 7\nretry: 25\n\n", + ]; + assert_eq!( + parse_fragments(fragments, 64, 128, 256).unwrap(), + vec![event("hé\ntwo", Some("final"), Some("7"), Some(25))] + ); + } + + #[test] + fn parser_clears_event_type_at_empty_data_dispatch_boundary() { + assert_eq!( + parse_fragments( + &[b"event: custom\nid: 7\nretry: 25\n\ndata: payload\n\n"], + 64, + 128, + 256 + ) + .unwrap(), + vec![event("payload", None, Some("7"), Some(25))] + ); + } + + #[test] + fn parser_clears_fragmented_event_type_at_crlf_boundaries() { + let fragments: &[&[u8]] = &[ + b"event: custom\r", + b"\nid: 7\r\nretry: 25\r", + b"\n\r\ndata: pay", + b"load\r\n\r", + b"\nevent: named\r\ndata: second\r\n\r\n", + b"data: next\r\n\r\n", + ]; + assert_eq!( + parse_fragments(fragments, 64, 128, 256).unwrap(), + vec![ + event("payload", None, Some("7"), Some(25)), + event("second", Some("named"), Some("7"), Some(25)), + event("next", None, Some("7"), Some(25)), + ] + ); + } + + #[test] + fn parser_uses_first_colon_removes_one_space_and_ignores_comments_unknown_fields() { + let input = b": comment\ndata:a:b\ndata: two\ndata: \nunknown: value\n\n"; + assert_eq!( + parse_fragments(&[input], 64, 128, 256).unwrap(), + vec![event("a:b\n two\n", None, None, None)] + ); + } + + #[test] + fn parser_handles_empty_fields_id_nul_and_retry_rules() { + let input = b"id: keep\nretry: 42\ndata: one\n\nretry: 99\n\nid:\nid: bad\0id\nretry: -1\nretry: 4x\nretry: 9223372036854775808\nevent:\ndata: two\n\n"; + assert_eq!( + parse_fragments(&[input], 64, 128, 512).unwrap(), + vec![ + event("one", None, Some("keep"), Some(42)), + event("two", Some(""), Some(""), Some(99)), + ] + ); + } + + #[test] + fn parser_persists_retry_state_across_empty_blocks_events_and_invalid_values() { + let input = b"retry:5000\n\ndata:ready\n\ndata:next\n\nretry:\nretry: -1\nretry: 5x\nretry: 9223372036854775808\n\ndata:still\n\n"; + assert_eq!( + parse_fragments(&[input], 64, 128, 512).unwrap(), + vec![ + event("ready", None, None, Some(5000)), + event("next", None, None, Some(5000)), + event("still", None, None, Some(5000)), + ] + ); + } + + #[test] + fn parser_discards_incomplete_event_at_eof_and_ignores_field_only_blocks() { + assert!( + parse_fragments(&[b"event: named\nid: x\n\ndata: tail"], 64, 128, 256) + .unwrap() + .is_empty() + ); + assert_eq!( + parse_fragments(&[b"id: x\n\ndata: complete\n\n"], 64, 128, 256).unwrap(), + vec![event("complete", None, Some("x"), None)] + ); + assert!( + parse_fragments(&[b"event: unused"], 64, 128, 256) + .unwrap() + .is_empty() + ); + } + + #[test] + fn parser_rejects_malformed_and_incomplete_utf8() { + for input in [ + b"data: \xff\n\n".as_slice(), + b"data: \xc3".as_slice(), + // A BOM prefix that never completes is still invalid UTF-8 and + // must surface from `finish` at EOF instead of being dropped. + b"\xef".as_slice(), + b"\xef\xbb".as_slice(), + ] { + assert!( + parse_fragments(&[input], 64, 128, 256) + .unwrap_err() + .contains("UTF-8") + ); + } + } + + #[test] + fn parser_enforces_exact_line_item_and_total_boundaries() { + assert_eq!( + parse_fragments(&[b"data: ab\n\n"], 8, 2, 10).unwrap(), + vec![event("ab", None, None, None)] + ); + assert!( + parse_fragments(&[b"data: abc\n\n"], 8, 3, 12) + .unwrap_err() + .contains("line") + ); + assert!( + parse_fragments(&[b"data: ab\ndata: c\n\n"], 16, 3, 64) + .unwrap_err() + .contains("item") + ); + assert!( + parse_fragments(&[b"data: ab\n\n"], 8, 2, 9) + .unwrap_err() + .contains("total") + ); + } + + #[test] + fn parser_rejects_a_single_fragment_before_unbounded_growth() { + let mut parser = SseParser::new(4, 16, 64); + assert!(parser.push(b"data: a very large fragment").is_err()); + } + + #[test] + fn parser_only_strips_a_bom_at_the_start_of_the_stream() { + assert_eq!( + parse_fragments( + &[b"data: first\n\ndata: \xef\xbb\xbfsecond\n\n"], + 64, + 128, + 256 + ) + .unwrap(), + vec![ + event("first", None, None, None), + event("\u{feff}second", None, None, None), + ] + ); + } +} diff --git a/src/builtins/runtime/mod.rs b/src/builtins/runtime/mod.rs index b0ce8c20..43dc9d62 100644 --- a/src/builtins/runtime/mod.rs +++ b/src/builtins/runtime/mod.rs @@ -3,10 +3,10 @@ use std::task::{Context, Poll}; use crate::builtins::BuiltinFunction; -#[cfg(feature = "async")] -use crate::vm::CaptureAsyncHostContext; #[allow(unused_imports)] use crate::vm::{CallOutcome, CallReturn, HostOpId, Value, Vm, VmError, VmResult}; +#[cfg(feature = "async")] +use crate::vm::{CaptureAsyncHostContext, HostFutureOutput}; mod aot; mod bytes; @@ -16,6 +16,8 @@ pub(crate) mod core; pub(crate) mod error; pub(crate) mod event; mod host; +#[cfg(feature = "http-client")] +pub(crate) mod http; #[cfg(not(target_arch = "wasm32"))] mod io; #[cfg(target_arch = "wasm32")] @@ -37,6 +39,8 @@ use io_wasm as io; pub(crate) use context::{RuntimeContext, RuntimeContextConfig, STREAM_EMIT_NAME}; pub use error::{RuntimeError, RuntimeErrorCode, RuntimeResult}; pub(crate) use event::{EventLimits, EventPayload}; +#[cfg(feature = "http-client")] +pub use http::{HttpConfig, HttpHostExt}; pub(crate) use io::IoState; #[cfg(not(target_arch = "wasm32"))] pub use io::{IoHostExt, IoPolicy}; diff --git a/src/lib.rs b/src/lib.rs index 4f1c3c34..2814527a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,6 +31,8 @@ pub use builtins::runtime::{ BorrowVmValue, FromVmValue, HostCallResult, IntoHostCallOutcome, TakeVmValue, arg, borrow_arg, return_one, take_arg, }; +#[cfg(all(feature = "runtime", feature = "http-client"))] +pub use builtins::runtime::{HttpConfig, HttpHostExt}; #[cfg(all(feature = "runtime", not(target_arch = "wasm32")))] pub use builtins::runtime::{IoHostExt, IoPolicy}; #[cfg(feature = "runtime")] @@ -45,7 +47,7 @@ pub use builtins::{ pub use bytecode::{ CallableEnvironment, CallableKind, CallablePrototype, CallableTarget, CallableValue, CaptureBindingMode, ExportedCallable, FunctionRegion, HostImport, OpCode, Program, - RootCallableBinding, ScriptFunction, TypeMap, Value, ValueType, + RootCallableBinding, ScriptFunction, TypeMap, Value, ValueType, VmMap, }; pub use host_api::{ FunctionNameError, HostApiBuilder, HostApiCatalog, HostApiCatalogError, HostApiFingerprint, diff --git a/src/vm/async_host/mod.rs b/src/vm/async_host/mod.rs index 8dda97ee..c6902038 100644 --- a/src/vm/async_host/mod.rs +++ b/src/vm/async_host/mod.rs @@ -23,6 +23,9 @@ use std::pin::Pin; use super::*; +pub(crate) mod stream; +pub(crate) use stream::{HostStreamAction, HostStreamDriver, HostStreamPoll}; + /// A completion closure that runs against the VM after the async call's /// future has resolved. pub type HostVmCompletion = Box VmResult + Send + 'static>; diff --git a/src/vm/async_host/stream.rs b/src/vm/async_host/stream.rs new file mode 100644 index 00000000..e1477719 --- /dev/null +++ b/src/vm/async_host/stream.rs @@ -0,0 +1,370 @@ +use std::task::{Context, Poll}; + +use crate::compiler::TypeSchema; +use crate::vm::host::WaitingHostOpSource; +use crate::vm::{CallOutcome, HostOpId, Value, Vm, VmError, VmResult, VmStatus}; + +/// The result of one host-side producer poll for a callable stream. +/// +/// This is a host-only embedding extension point. It does not expose a stream +/// handle or polling operation to scripts. A [`HostStreamDriver::poll_next`] +/// call may yield at most one `Item`; the VM serializes that item with its +/// script callback before polling the producer again. +#[cfg_attr(not(feature = "http-client"), allow(dead_code))] +#[derive(Debug)] +pub(crate) enum HostStreamPoll { + /// Deliver one producer item to the script callback. + Item(Value), + /// Finish the stream and return the supplied summary to the script call. + Complete(Value), +} + +/// The host driver's response to one completed script callback. +/// +/// Values returned by the callback remain inside the host embedding boundary: +/// no action handle is exposed to scripts. +#[cfg_attr(not(feature = "http-client"), allow(dead_code))] +#[derive(Debug)] +pub(crate) enum HostStreamAction { + /// Continue by returning control to producer polling. + Continue, + /// Finish the stream and return the supplied summary to the script call. + Complete(Value), +} + +/// Host-only producer integration for a VM-serialized callable stream. +/// +/// The VM always validates the callback's callable provenance and arity before +/// installing a driver. When its metadata is [`TypeSchema::Callable`], it also +/// validates the argument and result schemas against `fn(map) -> map`. Scripts +/// receive ordinary callback items and a final value; they never receive a +/// stream handle or a producer poll API. +/// +/// Implementors must observe these contracts: +/// +/// - [`poll_next`](Self::poll_next) yields at most one item per call and must +/// never re-enter the VM. +/// - [`apply_action`](Self::apply_action) takes ownership of the callback's +/// returned [`Value`], validates it as a driver-specific action, and must not +/// poll the producer. +/// - Dropping the driver is terminal resource cleanup after normal completion, +/// cancellation, or error. Only an early drop represents cancellation, and a +/// `Drop` implementation cannot infer the terminal reason; it must release +/// producer resources without requiring another poll. +#[cfg_attr(not(feature = "http-client"), allow(dead_code))] +pub(crate) trait HostStreamDriver: Send + 'static { + /// Polls the producer for at most one item or its final summary. + fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll>; + + /// Validates and applies one callback-returned action value. + fn apply_action(&mut self, action: Value) -> VmResult; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum HostStreamPhase { + AwaitItem, + RunCallback, +} + +pub(crate) struct HostStreamContinuation { + pub(crate) op_id: HostOpId, + pub(crate) callback: Value, + pub(crate) item: Option, + pub(crate) phase: HostStreamPhase, + pub(crate) parent_stack_base: usize, + pub(crate) parent_frame_count: usize, + pub(crate) parent_ip: usize, +} + +impl Vm { + /// Installs a host-only callable stream and suspends the current VM call. + /// + /// This Rust embedding API does not create a script-visible handle. The VM + /// always validates that `callback` is a callable owned by this VM and has + /// arity one. When its metadata is [`TypeSchema::Callable`], the VM also + /// validates its argument and result schemas against `fn(map) -> map`. It + /// then owns the callback and driver until completion, cancellation, reset, + /// or error; removing the driver drops it to release producer resources. + /// + /// The driver contract is documented on [`HostStreamDriver`]. In + /// particular, producer polling and callback action application stay + /// serialized and neither driver method may re-enter the VM. + #[cfg_attr(not(feature = "http-client"), allow(dead_code))] + pub(crate) fn submit_callable_stream( + &mut self, + callback: Value, + driver: impl HostStreamDriver, + ) -> VmResult { + self.validate_stream_callback_value(&callback)?; + if self.instance.host_stream.is_some() { + return Err(VmError::HostError( + "vm already owns an active callable stream".to_string(), + )); + } + let op_id = self.allocate_host_op_id(); + self.host.stream_drivers.insert(op_id, Box::new(driver)); + self.instance.host_stream = Some(HostStreamContinuation { + op_id, + callback, + item: None, + phase: HostStreamPhase::AwaitItem, + parent_stack_base: self.instance.stack.len(), + parent_frame_count: self.instance.execution_frames.len(), + parent_ip: self.instance.ip, + }); + Ok(CallOutcome::Pending(op_id)) + } + + /// Validates that a callback value is a callable owned by this VM with + /// arity one and, when its metadata is available, `fn(map) -> map` shape. + pub fn validate_stream_callback_value(&self, callback: &Value) -> VmResult<()> { + let Value::Callable(callable) = callback else { + return Err(VmError::TypeMismatch("callable")); + }; + if !self.owns_callable(callback) { + return Err(VmError::InvalidCallable); + } + let prototype = self + .program + .callable_prototypes + .get(callable.prototype_id as usize) + .ok_or(VmError::InvalidCallablePrototype(callable.prototype_id))?; + if prototype.arity != 1 { + return Err(VmError::CallableArityMismatch { + prototype_id: callable.prototype_id, + expected: 1, + got: prototype.arity, + }); + } + if let Some(TypeSchema::Callable { params, result }) = &prototype.schema + && (!matches!(params.as_slice(), [TypeSchema::Map(_)]) + || !matches!(result.as_ref(), TypeSchema::Map(_))) + { + return Err(VmError::TypeMismatch("fn(map) -> map")); + } + Ok(()) + } + + /// Records the resume instruction pointer of the host call that admitted + /// the stream, so a later callback completion can restore the parent + /// frame position. + pub(crate) fn record_callable_stream_resume_ip(&mut self, op_id: HostOpId, resume_ip: usize) { + if let Some(stream) = self.instance.host_stream.as_mut() + && stream.op_id == op_id + { + stream.parent_ip = resume_ip; + } + } + + /// Cancels the active callable stream, dropping the driver (releasing + /// producer resources) and the owned callback/item values. + pub(crate) fn cancel_callable_stream(&mut self) { + if let Some(stream) = self.instance.host_stream.take() { + self.host.stream_drivers.remove(&stream.op_id); + if let Some(item) = stream.item { + self.drop_value_with_contract(item); + } + self.drop_value_with_contract(stream.callback); + } + } + + /// Polls the active callable stream's producer, delivering items into the + /// script callback and applying its returned action. + pub(crate) fn poll_callable_stream( + &mut self, + op_id: HostOpId, + cx: &mut Context<'_>, + ) -> Poll> { + if self + .instance + .host_stream + .as_ref() + .map(|stream| stream.phase) + != Some(HostStreamPhase::AwaitItem) + { + return Poll::Ready(Err(VmError::InvalidFrameState( + "callable stream producer polled during callback", + ))); + } + let polled = match self.host.stream_drivers.get_mut(&op_id) { + Some(driver) => driver.poll_next(cx), + None => { + return Poll::Ready(Err(VmError::HostError(format!( + "missing callable stream driver {op_id}" + )))); + } + }; + match polled { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(error)) => { + self.abort_callable_stream(); + Poll::Ready(Err(error)) + } + Poll::Ready(Ok(HostStreamPoll::Complete(summary))) => { + self.finish_callable_stream(summary); + Poll::Ready(Ok(())) + } + Poll::Ready(Ok(HostStreamPoll::Item(item))) => { + self.instance.waiting_host_op = None; + if let Some(stream) = self.instance.host_stream.as_mut() { + stream.phase = HostStreamPhase::RunCallback; + stream.item = Some(item); + } + match self.start_callable_stream_callback() { + Ok(VmStatus::Halted) => match self.finish_callable_stream_callback() { + Ok(VmStatus::Halted) => Poll::Ready(Ok(())), + Ok(VmStatus::Waiting(_)) => { + cx.waker().wake_by_ref(); + Poll::Pending + } + Ok(VmStatus::Yielded) => Poll::Ready(Ok(())), + Err(error) => Poll::Ready(Err(error)), + }, + Ok(VmStatus::Yielded | VmStatus::Waiting(_)) => Poll::Ready(Ok(())), + Err(error) => { + self.abort_callable_stream(); + Poll::Ready(Err(error)) + } + } + } + } + } + + fn start_callable_stream_callback(&mut self) -> VmResult { + let (callback, item) = { + let stream = self + .instance + .host_stream + .as_mut() + .ok_or(VmError::InvalidFrameState( + "missing callable stream continuation", + ))?; + ( + stream.callback.clone(), + stream + .item + .take() + .ok_or(VmError::InvalidFrameState("missing callable stream item"))?, + ) + }; + let operand_stack_base = self.instance.stack.len(); + let Value::Callable(callable) = callback else { + return Err(VmError::InvalidCallable); + }; + let outcome = self.enter_script_frame( + callable.prototype_id, + Some(callable), + vec![item], + operand_stack_base, + None, + crate::vm::instance::FrameContinuation::ReturnToHost, + )?; + match outcome { + crate::vm::ExecOutcome::Continue => self.run_internal(None, false), + crate::vm::ExecOutcome::Halted => Ok(VmStatus::Halted), + crate::vm::ExecOutcome::Yielded => Ok(VmStatus::Yielded), + crate::vm::ExecOutcome::Waiting(id) => Ok(VmStatus::Waiting(id)), + } + } + + /// Resumes the stream after a `run` step that halted with a callback + /// frame on top: consumes the callback's returned action and either + /// continues producer polling or completes the stream. + pub(crate) fn resume_callable_stream_after_run( + &mut self, + status: VmStatus, + ) -> VmResult { + if self + .instance + .host_stream + .as_ref() + .is_none_or(|stream| stream.phase != HostStreamPhase::RunCallback) + || status != VmStatus::Halted + { + return Ok(status); + } + self.finish_callable_stream_callback() + } + + /// Aborts the stream when a `run` step failed while a callback frame was + /// on top, releasing interpreter state and producer resources. + pub(crate) fn abort_callable_stream_on_run_error(&mut self) { + if self + .instance + .host_stream + .as_ref() + .is_some_and(|stream| stream.phase == HostStreamPhase::RunCallback) + { + self.abort_callable_stream(); + } + } + + fn finish_callable_stream_callback(&mut self) -> VmResult { + let Some(action) = self.instance.host_return.take() else { + self.abort_callable_stream(); + return Err(VmError::InvalidFrameState( + "callable stream callback returned no action", + )); + }; + let op_id = self + .instance + .host_stream + .as_ref() + .ok_or(VmError::InvalidFrameState( + "missing callable stream continuation", + ))? + .op_id; + if let Some(stream) = self.instance.host_stream.as_ref() { + self.instance.ip = stream.parent_ip; + } + let applied = self + .host + .stream_drivers + .get_mut(&op_id) + .ok_or_else(|| VmError::HostError(format!("missing callable stream driver {op_id}")))? + .apply_action(action); + match applied { + Ok(HostStreamAction::Continue) => { + if let Some(stream) = self.instance.host_stream.as_mut() { + stream.phase = HostStreamPhase::AwaitItem; + } + self.set_waiting_host_op(op_id, WaitingHostOpSource::CallableStream)?; + Ok(VmStatus::Waiting(op_id)) + } + Ok(HostStreamAction::Complete(summary)) => { + self.finish_callable_stream(summary); + Ok(VmStatus::Halted) + } + Err(error) => { + self.abort_callable_stream(); + Err(error) + } + } + } + + fn finish_callable_stream(&mut self, summary: Value) { + let Some(stream) = self.instance.host_stream.take() else { + return; + }; + self.host.stream_drivers.remove(&stream.op_id); + self.instance.waiting_host_op = None; + self.drop_value_with_contract(stream.callback); + if let Some(item) = stream.item { + self.drop_value_with_contract(item); + } + self.instance.stack.push(summary); + } + + fn abort_callable_stream(&mut self) { + let Some(stream) = self.instance.host_stream.take() else { + return; + }; + self.host.stream_drivers.remove(&stream.op_id); + self.instance.waiting_host_op = None; + self.abort_host_invocation(stream.parent_stack_base, stream.parent_frame_count); + self.drop_value_with_contract(stream.callback); + if let Some(item) = stream.item { + self.drop_value_with_contract(item); + } + } +} diff --git a/src/vm/host.rs b/src/vm/host.rs index 692aa37f..c23a7785 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -780,6 +780,11 @@ pub(super) enum WaitingHostOpSource { BuiltinIo, #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] BuiltinSqlite, + /// A callable-stream continuation (a host-only producer whose items are + /// serialized through a script callback). Polling and cancellation are + /// handled by the stream driver machinery rather than a bridge or a + /// runtime-owned operation. + CallableStream, } struct NoopWake; @@ -1177,6 +1182,13 @@ impl Vm { } pub(super) fn cancel_waiting_host_op(&mut self) { + self.cancel_waiting_host_op_with_reason(OperationCancelReason::Requested); + } + + /// Cancels the currently waiting host op (if any), forwarding a typed + /// reason so the bridge/stream driver can distinguish an explicit request + /// from a reset, deadline, or drop. + pub(super) fn cancel_waiting_host_op_with_reason(&mut self, reason: OperationCancelReason) { let Some(waiting) = self.instance.waiting_host_op.take() else { return; }; @@ -1184,7 +1196,7 @@ impl Vm { WaitingHostOpSource::HostBridge => { self.host.submitted_host_ops.remove(&waiting.op_id); if let Some(bridge) = self.host.async_bridge.as_mut() { - bridge.cancel_op(waiting.op_id); + bridge.cancel_op_with_reason(waiting.op_id, reason); } } WaitingHostOpSource::BuiltinIo => { @@ -1194,6 +1206,9 @@ impl Vm { WaitingHostOpSource::BuiltinSqlite => { crate::builtins::runtime::cancel_builtin_sqlite_op(self, waiting.op_id); } + WaitingHostOpSource::CallableStream => { + self.cancel_callable_stream(); + } } } @@ -1210,6 +1225,10 @@ impl Vm { return Poll::Ready(Ok(())); }; + if waiting.source == WaitingHostOpSource::CallableStream { + return self.poll_callable_stream(waiting.op_id, cx); + } + // The HostBridge arm produces a `HostFutureOutput` (so a submitted // future's completion closure can run against the VM); the runtime // builtin arms produce an already-finished `CallReturn`. @@ -1248,6 +1267,8 @@ impl Vm { crate::builtins::runtime::poll_builtin_sqlite_op(self, waiting.op_id, cx) .map(|result| result.map(HostFutureOutput::Return)) } + // Handled above through `poll_callable_stream`; unreachable here. + WaitingHostOpSource::CallableStream => unreachable!(), }; match poll_result { @@ -1457,7 +1478,10 @@ impl Vm { crate::builtins::runtime::BuiltinCallOutcome::Pending(op_id) => { self.instance.stack.truncate(arg_start); let resume_ip = self.call_resume_ip(call_ip)?; - if self.host.submitted_host_ops.contains(&op_id) { + self.record_callable_stream_resume_ip(op_id, resume_ip); + if self.host.stream_drivers.contains_key(&op_id) { + self.set_waiting_host_op(op_id, WaitingHostOpSource::CallableStream)?; + } else if self.host.submitted_host_ops.contains(&op_id) { if let Err(error) = self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge) { @@ -1916,7 +1940,12 @@ impl Vm { saved_stack.append(&mut host_stack); self.instance.stack = saved_stack; let resume_ip = self.call_resume_ip(call_ip)?; - self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; + self.record_callable_stream_resume_ip(op_id, resume_ip); + if self.host.stream_drivers.contains_key(&op_id) { + self.set_waiting_host_op(op_id, WaitingHostOpSource::CallableStream)?; + } else { + self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; + } self.instance.ip = resume_ip; Ok(HostCallExecOutcome::Pending(op_id)) } @@ -2031,7 +2060,12 @@ impl Vm { CallOutcome::Pending(op_id) => { self.instance.stack.truncate(arg_start); let resume_ip = self.call_resume_ip(call_ip)?; - self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; + self.record_callable_stream_resume_ip(op_id, resume_ip); + if self.host.stream_drivers.contains_key(&op_id) { + self.set_waiting_host_op(op_id, WaitingHostOpSource::CallableStream)?; + } else { + self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; + } self.instance.ip = resume_ip; Ok(HostCallExecOutcome::Pending(op_id)) } @@ -2092,7 +2126,12 @@ impl Vm { CallOutcome::Pending(op_id) => { self.instance.stack.truncate(arg_start); let resume_ip = self.call_resume_ip(call_ip)?; - self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; + self.record_callable_stream_resume_ip(op_id, resume_ip); + if self.host.stream_drivers.contains_key(&op_id) { + self.set_waiting_host_op(op_id, WaitingHostOpSource::CallableStream)?; + } else { + self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; + } self.instance.ip = resume_ip; Ok(HostCallExecOutcome::Pending(op_id)) } diff --git a/src/vm/host_runtime.rs b/src/vm/host_runtime.rs index 99658cde..9e38111e 100644 --- a/src/vm/host_runtime.rs +++ b/src/vm/host_runtime.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use crate::builtins::runtime::IoState; #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] use crate::builtins::runtime::SqliteState; +use crate::vm::async_host::HostStreamDriver; use crate::vm::execution_scope::ExecutionScope; use crate::vm::host::{HostAsyncBridge, HostOpId, VmHostFunction}; @@ -82,6 +83,11 @@ pub(crate) struct HostRuntime { /// that are still pending. These route to the bridge's /// `poll_submitted_op` instead of a runtime-owned operation driver. pub(crate) submitted_host_ops: HashSet, + /// Callable-stream drivers keyed by their host-operation id. A driver is + /// installed by `Vm::submit_callable_stream` and removed when its stream + /// completes, is cancelled, or errors; dropping a driver releases its + /// producer resources. + pub(crate) stream_drivers: HashMap>, } impl HostRuntime { @@ -116,6 +122,7 @@ impl HostRuntime { allowed_host_function_slots: Vec::new(), allow_default_host_fallback: true, submitted_host_ops: HashSet::new(), + stream_drivers: HashMap::new(), } } @@ -138,6 +145,18 @@ impl HostRuntime { .and_then(|state| state.downcast_ref::()) } + /// Mutable access to host-owned typed policy/configuration state, if any. + #[cfg(feature = "http-client")] + pub(crate) fn host_function_state_mut(&mut self) -> Option<&mut T> + where + T: Send + Sync + 'static, + { + self.host_function_state + .get_mut(&TypeId::of::()) + .map(|state| Arc::get_mut(state).expect("http host state is uniquely owned")) + .and_then(|state| state.downcast_mut::()) + } + /// Removes host-owned typed policy/configuration state. pub(crate) fn remove_host_function_state(&mut self) -> Option> where @@ -157,13 +176,35 @@ impl HostRuntime { /// in-flight IO operation and closing every IO handle/process resource /// before the new scope starts. Used by `Vm::reset_for_reuse` so IO /// retirement goes through the generic scope lifecycle. + /// + /// The typed policy/configuration store is cleared, except the persistent + /// HTTP host configuration, which is a *module-level* policy that survives + /// scope reset so an embedder's accepted hosts/schemes/limits remain in + /// force across `reset_for_reuse` (only the in-flight connection permits + /// and live streams are retired, never the configured policy). pub(crate) fn reset_execution_scope(&mut self) { self.io_state = IoState::default(); #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] { self.sqlite_state = SqliteState::default(); } - self.host_function_state.clear(); + #[cfg(feature = "http-client")] + { + let http_config = self + .host_function_state::() + .cloned(); + self.host_function_state.clear(); + if let Some(config) = http_config { + self.host_function_state.insert( + std::any::TypeId::of::(), + std::sync::Arc::new(config), + ); + } + } + #[cfg(not(feature = "http-client"))] + { + self.host_function_state.clear(); + } self.execution_scope = ExecutionScope::new() .expect("host runtime execution-scope identity space must be available"); } diff --git a/src/vm/host_stream_tests.rs b/src/vm/host_stream_tests.rs new file mode 100644 index 00000000..82c5988c --- /dev/null +++ b/src/vm/host_stream_tests.rs @@ -0,0 +1,1125 @@ +use std::collections::{HashMap, VecDeque}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll, Wake, Waker}; + +use super::{HostStreamAction, HostStreamDriver, HostStreamPoll}; +use crate::vm::operation::reason::OperationCancelReason; +use crate::{ + CallOutcome, CallReturn, HostAsyncBridge, HostFunction, HostFuture, HostFutureOutput, HostOpId, + InvocationError, InvocationItem, InvocationPoll, JitConfig, Value, Vm, VmError, VmMap, + VmResult, VmStatus, compile_source, +}; + +fn map(entries: impl IntoIterator) -> Value { + Value::Map(Arc::new(VmMap::from_entries( + entries + .into_iter() + .map(|(key, value)| (string(key), value)) + .collect(), + ))) +} + +fn string(value: &str) -> Value { + Value::String(Arc::new(value.to_string())) +} + +fn map_field<'a>(value: &'a Value, name: &str) -> Option<&'a Value> { + let Value::Map(entries) = value else { + return None; + }; + entries.get(&string(name)) +} + +#[derive(Default)] +struct NoopWake; + +impl Wake for NoopWake { + fn wake(self: Arc) {} +} + +fn context() -> Context<'static> { + let waker = Waker::from(Arc::new(NoopWake)); + Context::from_waker(Box::leak(Box::new(waker))) +} + +#[derive(Default)] +struct CountingWake(AtomicUsize); + +impl Wake for CountingWake { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + + fn wake_by_ref(self: &Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +struct SyntheticDriver { + items: VecDeque, + polls: Arc, + applied: Arc, + stopped: Arc, + producer_error: bool, +} + +impl Drop for SyntheticDriver { + fn drop(&mut self) { + self.stopped.fetch_add(1, Ordering::SeqCst); + } +} + +impl HostStreamDriver for SyntheticDriver { + fn poll_next(&mut self, _cx: &mut Context<'_>) -> Poll> { + self.polls.fetch_add(1, Ordering::SeqCst); + if self.producer_error { + return Poll::Ready(Err(VmError::HostError( + "synthetic producer failed".to_string(), + ))); + } + match self.items.pop_front() { + Some(item) => Poll::Ready(Ok(HostStreamPoll::Item(item))), + None => Poll::Ready(Ok(HostStreamPoll::Complete(map([ + ("outcome", string("eof")), + ( + "items", + Value::Int(self.applied.load(Ordering::SeqCst) as i64), + ), + ])))), + } + } + + fn apply_action(&mut self, action: Value) -> Result { + let Some(Value::String(action)) = map_field(&action, "action") else { + return Err(VmError::HostError( + "stream callback action must be a map with string 'action'".to_string(), + )); + }; + self.applied.fetch_add(1, Ordering::SeqCst); + match action.as_str() { + "continue" => Ok(HostStreamAction::Continue), + "stop" => Ok(HostStreamAction::Complete(map([ + ("outcome", string("stopped")), + ( + "items", + Value::Int(self.applied.load(Ordering::SeqCst) as i64), + ), + ]))), + other => Err(VmError::HostError(format!( + "invalid synthetic stream action '{other}'" + ))), + } + } +} + +struct DropOnlyDriver { + stopped: Arc, +} + +impl Drop for DropOnlyDriver { + fn drop(&mut self) { + self.stopped.fetch_add(1, Ordering::SeqCst); + } +} + +impl HostStreamDriver for DropOnlyDriver { + fn poll_next(&mut self, _cx: &mut Context<'_>) -> Poll> { + panic!("rejected driver must never be polled") + } + + fn apply_action(&mut self, _action: Value) -> VmResult { + panic!("rejected driver must never receive an action") + } +} + +struct PendingProducerDriver { + polls: Arc, + applied: Arc, + stopped: Arc, +} + +impl Drop for PendingProducerDriver { + fn drop(&mut self) { + self.stopped.fetch_add(1, Ordering::SeqCst); + } +} + +impl HostStreamDriver for PendingProducerDriver { + fn poll_next(&mut self, _cx: &mut Context<'_>) -> Poll> { + self.polls.fetch_add(1, Ordering::SeqCst); + Poll::Pending + } + + fn apply_action(&mut self, _action: Value) -> VmResult { + self.applied.fetch_add(1, Ordering::SeqCst); + panic!("a pending producer must never receive a callback action") + } +} + +struct PendingProducerHost { + polls: Arc, + applied: Arc, + stopped: Arc, + op_id: Arc, +} + +impl HostFunction for PendingProducerHost { + fn call(&mut self, vm: &mut Vm, args: &[Value]) -> VmResult { + let [callback] = args else { + return Err(VmError::HostError("expected one callback".to_string())); + }; + let outcome = vm.submit_callable_stream( + callback.clone(), + PendingProducerDriver { + polls: Arc::clone(&self.polls), + applied: Arc::clone(&self.applied), + stopped: Arc::clone(&self.stopped), + }, + )?; + if let CallOutcome::Pending(op_id) = outcome { + self.op_id.store(op_id as usize, Ordering::SeqCst); + } + Ok(outcome) + } +} + +struct SyntheticStreamHost { + polls: Arc, + applied: Arc, + stopped: Arc, + invalid_first: bool, + producer_error: bool, +} + +struct YieldOnceHost(bool); + +impl HostFunction for YieldOnceHost { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> Result { + if !self.0 { + self.0 = true; + Ok(CallOutcome::Yield) + } else { + Ok(CallOutcome::Return(vec![Value::Null].into())) + } + } +} + +struct YieldForeverHost(Arc); + +impl HostFunction for YieldForeverHost { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(CallOutcome::Yield) + } +} + +struct WaitHost; + +impl HostFunction for WaitHost { + fn call(&mut self, vm: &mut Vm, _args: &[Value]) -> Result { + vm.submit_host_future(Box::pin(std::future::pending())) + } +} + +struct RuntimeExitHost; + +impl HostFunction for RuntimeExitHost { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> Result { + Ok(CallOutcome::Halt) + } +} + +#[derive(Default)] +struct PendingBridge { + futures: HashMap, +} + +impl HostAsyncBridge for PendingBridge { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + self.futures.insert(op_id, future); + Ok(()) + } + + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown host operation {op_id}" + )))) + } + + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + cx: &mut Context<'_>, + ) -> Poll> { + self.futures.get_mut(&op_id).map_or( + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))), + |future| future.as_mut().poll(cx), + ) + } + + fn cancel_op(&mut self, op_id: HostOpId) { + self.futures.remove(&op_id); + } +} + +#[derive(Default)] +struct CancellationLog { + pending: HashMap, + cancellations: Vec<(HostOpId, OperationCancelReason)>, +} + +struct RecordingPendingBridge(Arc>); + +impl HostAsyncBridge for RecordingPendingBridge { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + self.0 + .lock() + .expect("cancellation log lock") + .pending + .insert(op_id, future); + Ok(()) + } + + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown host operation {op_id}" + )))) + } + + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + if self + .0 + .lock() + .expect("cancellation log lock") + .pending + .contains_key(&op_id) + { + Poll::Pending + } else { + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))) + } + } + + fn cancel_op_with_reason(&mut self, op_id: HostOpId, reason: OperationCancelReason) { + let mut log = self.0.lock().expect("cancellation log lock"); + log.pending.remove(&op_id); + log.cancellations.push((op_id, reason)); + } +} + +struct ErrorAfterYieldHost(bool); + +impl HostFunction for ErrorAfterYieldHost { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> Result { + if !self.0 { + self.0 = true; + Ok(CallOutcome::Yield) + } else { + Err(VmError::HostError( + "callback resumed into failure".to_string(), + )) + } + } +} + +impl HostFunction for SyntheticStreamHost { + fn call(&mut self, vm: &mut Vm, args: &[Value]) -> Result { + let [callback] = args else { + return Err(VmError::HostError("expected one callback".to_string())); + }; + let driver = SyntheticDriver { + items: (1..=4) + .map(|number| { + map([ + ("kind", string("item")), + ("n", Value::Int(number)), + ( + "action", + string(if self.invalid_first && number == 1 { + "invalid" + } else if number == 3 { + "stop" + } else { + "continue" + }), + ), + ]) + }) + .collect(), + polls: Arc::clone(&self.polls), + applied: Arc::clone(&self.applied), + stopped: Arc::clone(&self.stopped), + producer_error: self.producer_error, + }; + let outcome = vm.submit_callable_stream(callback.clone(), driver)?; + Ok(outcome) + } +} + +fn setup(source: &str) -> (Vm, Arc, Arc, Arc) { + let compiled = compile_source(source).expect("stream source should compile"); + let polls = Arc::new(AtomicUsize::new(0)); + let applied = Arc::new(AtomicUsize::new(0)); + let stopped = Arc::new(AtomicUsize::new(0)); + let mut vm = Vm::new(compiled.program); + vm.set_async_bridge(Box::new(PendingBridge::default())); + for function in compiled.functions { + match function.name.as_str() { + "synthetic_stream" | "synthetic_invalid" | "synthetic_error" => { + let invalid_first = function.name == "synthetic_invalid"; + let producer_error = function.name == "synthetic_error"; + vm.register_function(Box::new(SyntheticStreamHost { + polls: Arc::clone(&polls), + applied: Arc::clone(&applied), + stopped: Arc::clone(&stopped), + invalid_first, + producer_error, + })); + } + "yield_once" => { + vm.register_function(Box::new(YieldOnceHost(false))); + } + "wait_once" => { + vm.register_function(Box::new(WaitHost)); + } + "error_after_yield" => { + vm.register_function(Box::new(ErrorAfterYieldHost(false))); + } + "runtime::exit" => { + vm.register_function(Box::new(RuntimeExitHost)); + } + other => panic!("unexpected host import {other}"), + } + } + (vm, polls, applied, stopped) +} + +fn poll_once(vm: &mut Vm) -> Poll> { + vm.poll_waiting_host_op(&mut context()) +} + +fn direct_callback_vm(source: &str, export: &str) -> (Vm, Value) { + let compiled = compile_source(source).expect("direct callback source should compile"); + let mut vm = Vm::new(compiled.program); + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + let callback = vm.resolve_exported_callable(export).unwrap(); + (vm, callback) +} + +#[test] +fn delivers_three_maps_to_a_closure_in_order_and_returns_summary() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + synthetic_stream(|item| item); + "#; + let (mut vm, polls, applied, stopped) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + for expected in 1..=3 { + let poll = poll_once(&mut vm); + if expected < 3 { + assert!(matches!(poll, Poll::Pending)); + } else { + assert!(matches!(poll, Poll::Ready(Ok(())))); + } + assert_eq!(polls.load(Ordering::SeqCst), expected); + assert_eq!(applied.load(Ordering::SeqCst), expected); + } + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + assert_eq!( + map_field(&vm.stack()[0], "outcome"), + Some(&string("stopped")) + ); + assert_eq!(map_field(&vm.stack()[0], "items"), Some(&Value::Int(3))); +} + +#[tokio::test(flavor = "current_thread")] +async fn ready_callbacks_self_wake_until_the_stream_reaches_a_terminal_result() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + synthetic_stream(|item| item); + "#; + let (mut vm, polls, applied, stopped) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + + tokio::time::timeout( + std::time::Duration::from_millis(100), + vm.await_waiting_host_op(), + ) + .await + .expect("ready producer and callback must make executor-driven progress") + .unwrap(); + + assert_eq!(polls.load(Ordering::SeqCst), 3); + assert_eq!(applied.load(Ordering::SeqCst), 3); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert_eq!(vm.run().unwrap(), VmStatus::Halted); +} + +#[test] +fn continuing_callback_returns_pending_after_scheduling_its_own_repoll() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + synthetic_stream(|item| item); + "#; + let (mut vm, polls, applied, _) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + let wake = Arc::new(CountingWake::default()); + let waker = Waker::from(Arc::clone(&wake)); + let mut cx = Context::from_waker(&waker); + + assert!(matches!(vm.poll_waiting_host_op(&mut cx), Poll::Pending)); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 1); + assert_eq!(wake.0.load(Ordering::SeqCst), 1); + vm.reset_for_reuse(); +} + +#[test] +fn producer_is_not_polled_until_callback_action_is_applied() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + synthetic_stream(|item| item); + "#; + let (mut vm, polls, applied, _) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + assert!(matches!(poll_once(&mut vm), Poll::Pending)); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 1); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + vm.reset_for_reuse(); +} + +#[test] +fn invalid_action_aborts_before_a_second_producer_poll() { + let source = r#" + fn synthetic_invalid(callback: fn(map) -> map) -> map; + synthetic_invalid(|item| item); + "#; + let (mut vm, polls, _, _) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + let Poll::Ready(Err(VmError::HostError(message))) = poll_once(&mut vm) else { + panic!("invalid action should fail immediately") + }; + assert!(message.contains("invalid synthetic stream action")); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert!(vm.waiting_host_op_id().is_none()); +} + +#[test] +fn producer_error_releases_the_driver_and_clears_stream_waiting_state() { + let source = r#" + fn synthetic_error(callback: fn(map) -> map) -> map; + synthetic_error(|item| item); + "#; + let (mut vm, polls, applied, stopped) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + let Poll::Ready(Err(VmError::HostError(message))) = poll_once(&mut vm) else { + panic!("producer error should terminate the stream") + }; + assert_eq!(message, "synthetic producer failed"); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert!(vm.waiting_host_op_id().is_none()); +} + +#[test] +fn yielded_callback_resumes_before_the_producer_is_polled_again() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn yield_once(); + fn callback(item: map) -> map { yield_once(); item } + synthetic_stream(callback); + "#; + let (mut vm, polls, applied, _) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + assert!(matches!(poll_once(&mut vm), Poll::Ready(Ok(())))); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!( + vm.resume().unwrap(), + VmStatus::Waiting(vm.waiting_host_op_id().unwrap()) + ); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 1); +} + +#[test] +fn waiting_callback_resumes_to_the_outer_stream_continuation() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn wait_once(); + fn callback(item: map) -> map { wait_once(); item } + synthetic_stream(callback); + "#; + let (mut vm, polls, applied, _) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + assert!(matches!(poll_once(&mut vm), Poll::Ready(Ok(())))); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + let inner_id = vm.waiting_host_op_id().unwrap(); + assert_ne!(inner_id, 0); + vm.complete_host_op(inner_id, vec![Value::Null]).unwrap(); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 1); +} + +#[test] +fn resumed_callback_error_releases_the_stream_driver() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn error_after_yield(); + fn callback(item: map) -> map { error_after_yield(); item } + synthetic_stream(callback); + "#; + let (mut vm, polls, applied, stopped) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + assert!(matches!(poll_once(&mut vm), Poll::Ready(Ok(())))); + assert!( + matches!(vm.resume(), Err(VmError::HostError(message)) if message == "callback resumed into failure") + ); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert!(vm.waiting_host_op_id().is_none()); +} + +#[test] +fn runtime_exit_in_callback_retires_the_direct_stream_before_reporting_failure() { + let source = r#" + use runtime; + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn yield_once(); + pub fn callback(item: map) -> map { yield_once(); runtime::exit(); item } + synthetic_stream(callback); + "#; + let (mut vm, polls, applied, stopped) = setup(source); + let VmStatus::Waiting(op_id) = vm.run().unwrap() else { + panic!("stream should wait for its first producer item") + }; + + assert!(matches!(poll_once(&mut vm), Poll::Ready(Ok(())))); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 0); + + let VmError::InvalidFrameState(message) = vm + .resume() + .expect_err("runtime::exit in the callback should report a typed terminal failure") + else { + panic!("runtime::exit in the callback should report invalid callback completion") + }; + assert_eq!(message, "callable stream callback returned no action"); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert!(vm.waiting_host_op_id().is_none()); + + assert!(matches!(poll_once(&mut vm), Poll::Ready(Ok(())))); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + let late = vm + .complete_host_op(op_id, vec![Value::Null]) + .expect_err("a retired stream must reject late completion"); + assert!(late.to_string().contains("vm is not waiting on any op")); + + let callback = vm.resolve_exported_callable("callback").unwrap(); + let replacement_stopped = Arc::new(AtomicUsize::new(0)); + assert!(matches!( + vm.submit_callable_stream( + callback, + DropOnlyDriver { + stopped: Arc::clone(&replacement_stopped), + }, + ) + .unwrap(), + CallOutcome::Pending(_) + )); + vm.reset_for_reuse(); + assert_eq!(replacement_stopped.load(Ordering::SeqCst), 1); + assert_eq!(stopped.load(Ordering::SeqCst), 1); +} + +#[test] +fn runtime_exit_in_callback_is_a_fused_typed_invocation_failure() { + let source = r#" + use runtime; + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn callback(item: map) -> map { runtime::exit(); item } + pub fn run() -> map { synthetic_stream(callback) } + "#; + let (mut vm, polls, applied, stopped) = setup(source); + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + let callable = vm.resolve_exported_callable("run").unwrap(); + { + let mut invocation = vm.start_invocation(callable, vec![]).unwrap(); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Ready(Some(Err(InvocationError::Vm(VmError::InvalidFrameState( + "callable stream callback returned no action" + ))))) + )); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Ready(None) + )); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Ready(None) + )); + } + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert!(vm.waiting_host_op_id().is_none()); +} + +#[test] +fn invocation_cancellation_during_callback_wait_releases_the_stream_driver() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn wait_once(); + fn callback(item: map) -> map { wait_once(); item } + pub fn run() -> map { synthetic_stream(callback) } + "#; + let (mut vm, polls, applied, stopped) = setup(source); + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + let callable = vm.resolve_exported_callable("run").unwrap(); + { + let mut invocation = vm.start_invocation(callable, vec![]).unwrap(); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Pending + )); + assert_eq!(polls.load(Ordering::SeqCst), 1); + invocation.cancel(OperationCancelReason::Requested).unwrap(); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Ready(Some(Err(InvocationError::Cancelled( + OperationCancelReason::Requested + )))) + )); + } + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert!(vm.waiting_host_op_id().is_none()); +} + +#[test] +fn dropping_invocation_during_callback_wait_cancels_once_and_reuses_the_vm() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn wait_once(); + fn callback(item: map) -> map { wait_once(); item } + pub fn run() -> map { synthetic_stream(callback) } + pub fn plain() -> int { 42 } + "#; + let (mut vm, polls, applied, stopped) = setup(source); + let cancellations = Arc::new(Mutex::new(CancellationLog::default())); + vm.set_async_bridge(Box::new(RecordingPendingBridge(Arc::clone(&cancellations)))); + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + let run = vm.resolve_exported_callable("run").unwrap(); + let plain = vm.resolve_exported_callable("plain").unwrap(); + + { + let mut invocation = vm.start_invocation(run, vec![]).unwrap(); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Pending + )); + assert_eq!(polls.load(Ordering::SeqCst), 1); + } + + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + let cancelled = cancellations + .lock() + .expect("cancellation log lock") + .cancellations + .clone(); + assert_eq!(cancelled.len(), 1); + assert_eq!(cancelled[0].1, OperationCancelReason::Requested); + let late = vm + .complete_host_op(cancelled[0].0, vec![Value::Null]) + .expect_err("a cancelled callback wait must reject late completion"); + assert!(late.to_string().contains("vm is not waiting on any op")); + + let mut replacement = vm.start_invocation(plain, vec![]).unwrap(); + assert!(matches!( + replacement.poll_next().unwrap(), + InvocationPoll::Ready(Some(Ok(InvocationItem::Complete(Value::Int(42))))) + )); +} + +#[test] +fn dropping_invocation_during_callback_yield_does_not_resume_the_callback() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn yield_forever(); + fn callback(item: map) -> map { yield_forever(); item } + pub fn run() -> map { synthetic_stream(callback) } + pub fn plain() -> int { 42 } + "#; + let compiled = compile_source(source).expect("stream source should compile"); + let polls = Arc::new(AtomicUsize::new(0)); + let applied = Arc::new(AtomicUsize::new(0)); + let stopped = Arc::new(AtomicUsize::new(0)); + let callback_calls = Arc::new(AtomicUsize::new(0)); + let mut vm = Vm::new(compiled.program); + vm.set_async_bridge(Box::new(PendingBridge::default())); + for function in compiled.functions { + match function.name.as_str() { + "synthetic_stream" => { + vm.register_function(Box::new(SyntheticStreamHost { + polls: Arc::clone(&polls), + applied: Arc::clone(&applied), + stopped: Arc::clone(&stopped), + invalid_first: false, + producer_error: false, + })); + } + "yield_forever" => { + vm.register_function(Box::new(YieldForeverHost(Arc::clone(&callback_calls)))); + } + other => panic!("unexpected host import {other}"), + } + } + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + let run = vm.resolve_exported_callable("run").unwrap(); + let plain = vm.resolve_exported_callable("plain").unwrap(); + + { + let mut invocation = vm.start_invocation(run, vec![]).unwrap(); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Pending + )); + assert_eq!(callback_calls.load(Ordering::SeqCst), 2); + } + + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert_eq!(callback_calls.load(Ordering::SeqCst), 2); + let mut replacement = vm.start_invocation(plain, vec![]).unwrap(); + assert!(matches!( + replacement.poll_next().unwrap(), + InvocationPoll::Ready(Some(Ok(InvocationItem::Complete(Value::Int(42))))) + )); +} + +#[test] +fn explicit_cancel_then_drop_cancels_callback_wait_and_driver_once() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn wait_once(); + fn callback(item: map) -> map { wait_once(); item } + pub fn run() -> map { synthetic_stream(callback) } + "#; + let (mut vm, polls, applied, stopped) = setup(source); + let cancellations = Arc::new(Mutex::new(CancellationLog::default())); + vm.set_async_bridge(Box::new(RecordingPendingBridge(Arc::clone(&cancellations)))); + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + let run = vm.resolve_exported_callable("run").unwrap(); + + { + let mut invocation = vm.start_invocation(run, vec![]).unwrap(); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Pending + )); + invocation.cancel(OperationCancelReason::Deadline).unwrap(); + } + + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + let cancelled = &cancellations + .lock() + .expect("cancellation log lock") + .cancellations; + assert_eq!(cancelled.len(), 1); + assert_eq!(cancelled[0].1, OperationCancelReason::Deadline); +} + +#[test] +fn dropping_invocation_during_producer_wait_retires_the_stream_and_reuses_the_vm() { + let source = r#" + fn synthetic_pending(callback: fn(map) -> map) -> map; + pub fn run() -> map { synthetic_pending(|item| item) } + pub fn plain() -> int { 42 } + "#; + let compiled = compile_source(source).expect("stream source should compile"); + let polls = Arc::new(AtomicUsize::new(0)); + let applied = Arc::new(AtomicUsize::new(0)); + let stopped = Arc::new(AtomicUsize::new(0)); + let op_id = Arc::new(AtomicUsize::new(0)); + let mut vm = Vm::new(compiled.program); + for function in compiled.functions { + match function.name.as_str() { + "synthetic_pending" => { + vm.register_function(Box::new(PendingProducerHost { + polls: Arc::clone(&polls), + applied: Arc::clone(&applied), + stopped: Arc::clone(&stopped), + op_id: Arc::clone(&op_id), + })); + } + other => panic!("unexpected host import {other}"), + } + } + assert_eq!(vm.run().unwrap(), VmStatus::Halted); + let run = vm.resolve_exported_callable("run").unwrap(); + let plain = vm.resolve_exported_callable("plain").unwrap(); + + { + let mut invocation = vm.start_invocation(run, vec![]).unwrap(); + assert!(matches!( + invocation.poll_next().unwrap(), + InvocationPoll::Pending + )); + assert_eq!(polls.load(Ordering::SeqCst), 1); + } + + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert_eq!(polls.load(Ordering::SeqCst), 1); + assert_eq!(applied.load(Ordering::SeqCst), 0); + let retired_op_id = op_id.load(Ordering::SeqCst) as HostOpId; + let late = vm + .complete_host_op(retired_op_id, vec![Value::Null]) + .expect_err("a retired invocation must reject late completion"); + assert!(late.to_string().contains("vm is not waiting on any op")); + + let mut replacement = vm + .start_invocation(plain, vec![]) + .expect("the vm must accept a replacement invocation immediately"); + assert!(matches!( + replacement.poll_next().unwrap(), + InvocationPoll::Ready(Some(Ok(InvocationItem::Complete(Value::Int(42))))) + )); +} + +#[test] +fn reset_and_shutdown_release_a_waiting_stream_driver() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + synthetic_stream(|item| item); + "#; + let (mut reset_vm, ..) = setup(source); + assert!(matches!(reset_vm.run().unwrap(), VmStatus::Waiting(_))); + reset_vm.reset_for_reuse(); + assert!(reset_vm.waiting_host_op_id().is_none()); + + let (mut shutdown_vm, ..) = setup(source); + assert!(matches!(shutdown_vm.run().unwrap(), VmStatus::Waiting(_))); + shutdown_vm.shutdown(); + assert!(shutdown_vm.waiting_host_op_id().is_none()); +} + +fn enter_callback_wait(vm: &mut Vm) { + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + assert!(matches!(poll_once(vm), Poll::Ready(Ok(())))); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); +} + +#[test] +fn reset_shutdown_and_drop_release_a_stream_during_callback_wait() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn wait_once(); + fn callback(item: map) -> map { wait_once(); item } + synthetic_stream(callback); + "#; + + let (mut reset_vm, _, _, reset_stopped) = setup(source); + enter_callback_wait(&mut reset_vm); + reset_vm.reset_for_reuse(); + assert_eq!(reset_stopped.load(Ordering::SeqCst), 1); + assert!(reset_vm.waiting_host_op_id().is_none()); + + let (mut shutdown_vm, _, _, shutdown_stopped) = setup(source); + enter_callback_wait(&mut shutdown_vm); + shutdown_vm.shutdown(); + assert_eq!(shutdown_stopped.load(Ordering::SeqCst), 1); + assert!(shutdown_vm.waiting_host_op_id().is_none()); + + let (mut dropped_vm, _, _, drop_stopped) = setup(source); + enter_callback_wait(&mut dropped_vm); + drop(dropped_vm); + assert_eq!(drop_stopped.load(Ordering::SeqCst), 1); +} + +#[test] +fn direct_submit_rejects_wrong_schema_before_admitting_the_driver() { + let (mut vm, callback) = + direct_callback_vm(r#"pub fn callback(item: int) -> int { item }"#, "callback"); + let stopped = Arc::new(AtomicUsize::new(0)); + let error = vm + .submit_callable_stream( + callback, + DropOnlyDriver { + stopped: Arc::clone(&stopped), + }, + ) + .expect_err("wrong callback schema must be rejected"); + assert!(matches!(error, VmError::TypeMismatch("fn(map) -> map"))); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert!(vm.waiting_host_op_id().is_none()); +} + +#[test] +fn direct_submit_rejects_foreign_callable_with_matching_prototype_metadata() { + let source = r#"pub fn callback(item: map) -> map { item }"#; + let (foreign_vm, foreign_callback) = direct_callback_vm(source, "callback"); + let (mut receiving_vm, receiving_callback) = direct_callback_vm(source, "callback"); + let (Value::Callable(foreign), Value::Callable(receiving)) = + (&foreign_callback, &receiving_callback) + else { + panic!("exports must be callables") + }; + assert_eq!(foreign.prototype_id, receiving.prototype_id); + let stopped = Arc::new(AtomicUsize::new(0)); + + let error = receiving_vm + .submit_callable_stream( + foreign_callback, + DropOnlyDriver { + stopped: Arc::clone(&stopped), + }, + ) + .expect_err("callable from another vm must be rejected"); + assert!(matches!(error, VmError::InvalidCallable)); + assert_eq!(stopped.load(Ordering::SeqCst), 1); + assert!(receiving_vm.waiting_host_op_id().is_none()); + drop(foreign_vm); +} + +#[test] +fn terminal_stream_rejects_late_completion_through_the_direct_vm_api() { + let (mut vm, callback) = + direct_callback_vm(r#"pub fn callback(item: map) -> map { item }"#, "callback"); + let polls = Arc::new(AtomicUsize::new(0)); + let applied = Arc::new(AtomicUsize::new(0)); + let stopped = Arc::new(AtomicUsize::new(0)); + let CallOutcome::Pending(op_id) = vm + .submit_callable_stream( + callback, + SyntheticDriver { + items: VecDeque::from([map([("action", string("stop"))])]), + polls, + applied, + stopped, + producer_error: false, + }, + ) + .unwrap() + else { + panic!("stream admission must return pending") + }; + + assert!(matches!(poll_once(&mut vm), Poll::Ready(Ok(())))); + let error = vm + .complete_host_op(op_id, vec![Value::Null]) + .expect_err("terminal stream must reject a late completion"); + assert!(error.to_string().contains("vm is not waiting on any op")); +} + +#[test] +fn callback_schema_accepts_closures_and_named_generic_functions() { + for source in [ + r#"fn synthetic_stream(callback: fn(map) -> map) -> map; synthetic_stream(|value| value);"#, + r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + fn identity(value: T) -> T { value } + synthetic_stream(identity); + "#, + ] { + compile_source(source).expect("typed callable should compile"); + } +} + +#[test] +fn callback_schema_mismatches_are_rejected_at_compile_time() { + for source in [ + r#"fn synthetic_stream(callback: fn(map) -> map) -> map; synthetic_stream(|value, extra| {action: "stop"});"#, + r#"fn synthetic_stream(callback: fn(map) -> map) -> map; synthetic_stream(|value: int| {action: "stop"});"#, + r#"fn synthetic_stream(callback: fn(map) -> map) -> map; synthetic_stream(|value| 1);"#, + ] { + assert!( + compile_source(source).is_err(), + "source unexpectedly compiled: {source}" + ); + } +} + +#[test] +fn dropping_vm_releases_a_waiting_stream_driver_once() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + synthetic_stream(|item| item); + "#; + let (mut vm, _polls, _applied, stopped) = setup(source); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + drop(vm); + assert_eq!(stopped.load(Ordering::SeqCst), 1); +} + +#[test] +fn interpreter_jit_and_aot_use_the_same_host_stream_continuation() { + let source = r#" + fn synthetic_stream(callback: fn(map) -> map) -> map; + let mut warm = 0; + while warm < 100 { + warm = warm + 1; + } + synthetic_stream(|item| item); + "#; + let mut backends = vec!["interpreter"]; + #[cfg(feature = "cranelift-jit")] + backends.extend(["jit", "aot"]); + for backend in backends { + let (mut vm, polls, applied, _stopped) = setup(source); + vm.set_jit_config(JitConfig { + enabled: backend == "jit", + hot_loop_threshold: 1, + max_trace_len: 128, + }); + if backend == "aot" { + vm.compile_aot().expect("aot compile should succeed"); + } + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + assert!(matches!(poll_once(&mut vm), Poll::Pending)); + assert!(matches!(poll_once(&mut vm), Poll::Pending)); + assert!(matches!(poll_once(&mut vm), Poll::Ready(Ok(())))); + assert_eq!(vm.run().unwrap(), VmStatus::Halted, "{backend}"); + assert_eq!(polls.load(Ordering::SeqCst), 3, "{backend}"); + assert_eq!(applied.load(Ordering::SeqCst), 3, "{backend}"); + if backend == "jit" && native_jit_supported() { + assert!( + vm.jit_native_exec_count() > 0, + "jit stream setup must execute a native hot path: {}", + vm.dump_jit_info() + ); + } + if backend == "aot" { + assert!(vm.aot_exec_count() > 0, "aot stream path must execute"); + } + } +} + +fn native_jit_supported() -> bool { + (cfg!(target_arch = "x86_64") + && (cfg!(target_os = "windows") || (cfg!(unix) && !cfg!(target_os = "macos")))) + || (cfg!(target_arch = "aarch64") + && (cfg!(target_os = "linux") || cfg!(target_os = "macos"))) +} diff --git a/src/vm/instance.rs b/src/vm/instance.rs index a6183d0b..9859515c 100644 --- a/src/vm/instance.rs +++ b/src/vm/instance.rs @@ -18,6 +18,7 @@ use std::sync::atomic::AtomicBool; use std::sync::{Arc, Weak}; use crate::bytecode::{CallableValue, Program, SharedCaptureCell, Value}; +use crate::vm::async_host::stream::HostStreamContinuation; use crate::vm::host::WaitingHostOp; use crate::vm::invocation::{InvocationPhase, InvocationState}; use crate::vm::map_iter::MapIteratorState; @@ -84,6 +85,7 @@ pub(crate) struct Instance { pub(crate) draining_queued_callables: bool, pub(crate) shutdown: bool, pub(super) waiting_host_op: Option, + pub(crate) host_stream: Option, pub(crate) last_yield_reason: Option, pub(crate) invocation: Option, pub(crate) map_iterators: Vec>>, @@ -121,6 +123,7 @@ impl Instance { draining_queued_callables: false, shutdown: false, waiting_host_op: None, + host_stream: None, last_yield_reason: None, invocation: None, map_iterators: Vec::new(), @@ -163,6 +166,7 @@ impl Instance { self.draining_queued_callables = false; self.shutdown = false; self.waiting_host_op = None; + self.host_stream = None; self.drop_invocation_state(); self.invocation = None; self.map_iterators.clear(); diff --git a/src/vm/invocation.rs b/src/vm/invocation.rs index 62bd49b4..5f98193d 100644 --- a/src/vm/invocation.rs +++ b/src/vm/invocation.rs @@ -145,7 +145,8 @@ impl Invocation<'_> { )); } state.cancel_reason = Some(reason); - self.vm.cancel_waiting_host_op(); + self.vm.cancel_waiting_host_op_with_reason(reason); + self.vm.cancel_callable_stream(); Ok(()) } } @@ -461,7 +462,8 @@ impl Vm { .as_ref() .map(|state| (state.stack_base, state.frame_count)) .unwrap_or((0, 0)); - self.cancel_waiting_host_op(); + self.cancel_waiting_host_op_with_reason(OperationCancelReason::Requested); + self.cancel_callable_stream(); self.abort_host_invocation(stack_base, frame_count); self.instance.drop_invocation_state(); } diff --git a/src/vm/mod.rs b/src/vm/mod.rs index ae6169d5..994b37ac 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -15,6 +15,8 @@ mod host; pub mod host_context; pub mod host_extension; mod host_runtime; +#[cfg(test)] +mod host_stream_tests; mod instance; pub mod invocation; pub(crate) mod jit; @@ -31,6 +33,8 @@ mod superinstructions; mod tests; pub use self::aot::AotArtifactError; pub use self::async_host::{CaptureAsyncHostContext, HostFuture, HostFutureOutput}; +#[cfg_attr(not(feature = "http-client"), allow(unused_imports))] +pub(crate) use self::async_host::{HostStreamAction, HostStreamDriver, HostStreamPoll}; pub use self::capability::{CapabilityProfile, CapabilityProfileBuilder}; use self::engine::Engine; pub use self::epoch::{EpochCheckpoint, EpochHandle}; @@ -730,6 +734,7 @@ impl Vm { /// dropped and replaced with a fresh one). pub fn reset_for_reuse(&mut self) { self.cancel_waiting_host_op(); + self.cancel_callable_stream(); self.host.reset_execution_scope(); self.run_ctx.reset_for_reuse(); self.instance.reset(&self.program); @@ -1011,20 +1016,35 @@ impl Vm { } pub fn run(&mut self) -> VmResult { - self.run_internal(None, true) + let status = match self.run_internal(None, true) { + Ok(status) => status, + Err(error) => { + self.abort_callable_stream_on_run_error(); + return Err(error); + } + }; + self.resume_callable_stream_after_run(status) } pub fn run_with_debugger( &mut self, debugger: &mut crate::debugger::Debugger, ) -> VmResult { - self.run_internal(Some(debugger), false) + let status = match self.run_internal(Some(debugger), false) { + Ok(status) => status, + Err(error) => { + self.abort_callable_stream_on_run_error(); + return Err(error); + } + }; + self.resume_callable_stream_after_run(status) } } impl Drop for Vm { fn drop(&mut self) { self.cancel_waiting_host_op(); + self.cancel_callable_stream(); self.instance.drop_cleanup(); // Live IO handles and in-flight IO operations are retired by the // `ExecutionScope`'s own `Drop`, which runs as part of `HostRuntime`. @@ -2758,7 +2778,14 @@ impl Vm { .map(|frame| &frame.continuation), Some(FrameContinuation::ReturnToHost) ); - self.run_internal(None, allow_jit) + let status = match self.run_internal(None, allow_jit) { + Ok(status) => status, + Err(error) => { + self.abort_callable_stream_on_run_error(); + return Err(error); + } + }; + self.resume_callable_stream_after_run(status) } pub fn stack(&self) -> &[Value] { @@ -2993,6 +3020,7 @@ impl Vm { pub fn shutdown(&mut self) { self.invalidate_callback_registries(); self.cancel_waiting_host_op(); + self.cancel_callable_stream(); self.instance.drop_invocation_state(); // Begin execution-scope shutdown (first-reason-wins; sealing the // operation registry) before tearing down interpreter state. diff --git a/tests/host_binding_generation_tests.rs b/tests/host_binding_generation_tests.rs index 4636c82d..e3db7554 100644 --- a/tests/host_binding_generation_tests.rs +++ b/tests/host_binding_generation_tests.rs @@ -11,6 +11,8 @@ use vm::{ BuiltinFunction, CapabilityProfile, HostFunctionRegistry, JitConfig, JitTraceTerminal, Value, Vm, VmStatus, compile_source, }; +#[cfg(feature = "http-client")] +use vm::{HostExecution, default_host_callables}; fn native_jit_supported() -> bool { (cfg!(target_arch = "x86_64") @@ -302,6 +304,69 @@ fn restricted_capabilities_disable_trace_jit_for_host_imports_and_builtins() { } } +#[cfg(feature = "http-client")] +#[test] +fn generated_http_imports_are_unique_typed_and_independently_capability_gated() { + const IMPORTS: [&str; 2] = ["http::client::request", "http::client::sse"]; + let callables = default_host_callables(); + for name in IMPORTS { + let discovered = callables + .iter() + .filter(|callable| callable.name == name) + .collect::>(); + assert_eq!(discovered.len(), 1, "{name} discovery count"); + let callable = discovered[0]; + assert_eq!(callable.signature.return_type, "map"); + if name == "http::client::request" { + assert_eq!(callable.signature.params.len(), 1); + assert_eq!(callable.signature.params[0].ty.display_label(), "map"); + } else { + assert_eq!(callable.signature.params.len(), 2); + assert_eq!(callable.signature.params[0].ty.display_label(), "map"); + assert_eq!( + callable.signature.params[1].ty.display_label(), + "fn(map) -> map" + ); + assert_eq!(callable.host_execution, HostExecution::MaySuspend); + } + } + + for mask in 0_u8..4 { + let mut builder = CapabilityProfile::builder(); + for (index, name) in IMPORTS.iter().enumerate() { + if mask & (1 << index) != 0 { + builder = builder.allow_host_import(*name); + } + } + let profile = builder.build(); + for (index, name) in IMPORTS.iter().enumerate() { + assert_eq!( + profile.allows_host_import(name), + mask & (1 << index) != 0, + "mask {mask:02b}, import {name}" + ); + } + + let source = r#" + use http; + fn callback(item: map) -> map { { action: "stop" } } + http::client::request({ url: "https://example.test/" }); + http::client::sse({ url: "https://example.test/" }, callback); + "#; + let compiled = compile_source(source).expect("HTTP imports should compile"); + let mut vm = Vm::new(compiled.program); + let mut registry = HostFunctionRegistry::new(); + registry.set_capability_profile(profile); + let result = registry.bind_vm_cached(&mut vm); + if mask == 0b11 { + result.expect("both explicit capabilities should bind"); + } else { + let error = result.expect_err("a missing HTTP capability must reject binding"); + assert!(error.to_string().contains("capability profile"), "{error}"); + } + } +} + #[test] fn capability_profile_fingerprint_uses_stable_callable_identities() { let first = CapabilityProfile::builder() diff --git a/tests/http_feature_gating_tests.rs b/tests/http_feature_gating_tests.rs new file mode 100644 index 00000000..e0973d7b --- /dev/null +++ b/tests/http_feature_gating_tests.rs @@ -0,0 +1,32 @@ +#[test] +fn http_callables_follow_the_http_client_feature_gate() { + for name in ["http::client::request", "http::client::sse"] { + let published = vm::default_host_callables() + .iter() + .any(|callable| callable.name == name); + assert_eq!(published, cfg!(feature = "http-client"), "{name}"); + } +} + +#[cfg(feature = "http-client")] +#[test] +fn sse_callable_metadata_has_exact_stream_schema() { + let callable = vm::default_host_callables() + .iter() + .find(|callable| callable.name == "http::client::sse") + .expect("SSE callable should be published"); + assert_eq!( + callable + .signature + .params + .iter() + .map(|param| (param.name, param.ty.display_label(), param.optional)) + .collect::>(), + [ + ("request", "map".to_string(), false), + ("on_event", "fn(map) -> map".to_string(), false), + ] + ); + assert_eq!(callable.signature.return_type, "map"); + assert_eq!(callable.host_execution, vm::HostExecution::MaySuspend); +} diff --git a/tests/vm/http_host_tests.rs b/tests/vm/http_host_tests.rs new file mode 100644 index 00000000..9b19796a --- /dev/null +++ b/tests/vm/http_host_tests.rs @@ -0,0 +1,718 @@ +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; +use std::thread; + +use vm::{ + CallOutcome, CallReturn, HostAsyncBridge, HostFunctionRegistry, HostFuture, HostFutureOutput, + HostOpId, HttpConfig, HttpHostExt, Program, Value, Vm, VmError, VmResult, VmStatus, + compile_source, +}; + +#[derive(Default)] +struct TokioHostDriver { + submitted: HashMap, +} + +impl HostAsyncBridge for TokioHostDriver { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + self.submitted.insert(op_id, future); + Ok(()) + } + + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown external host operation {op_id}" + )))) + } + + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + cx: &mut Context<'_>, + ) -> Poll> { + let poll = self.submitted.get_mut(&op_id).map_or_else( + || { + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))) + }, + |future| future.as_mut().poll(cx), + ); + if poll.is_ready() { + self.submitted.remove(&op_id); + } + poll + } + + fn cancel_op(&mut self, op_id: HostOpId) { + self.submitted.remove(&op_id); + } +} + +fn install_host_driver(vm: &mut Vm) { + vm.set_async_bridge(Box::::default()); +} + +fn build_request_program(url: String) -> Program { + compile_source(&format!( + r#" + use http; + http::client::request({{"method": "GET", "url": "{url}"}}); + "# + )) + .expect("HTTP request source should compile") + .program +} + +fn build_request_program_with_method(url: &str, method: &str) -> Program { + compile_source(&format!( + r#" + use http; + http::client::request({{"method": "{method}", "url": "{url}", "body": "payload"}}); + "# + )) + .expect("HTTP request source should compile") + .program +} + +fn local_http_config(port: u16) -> HttpConfig { + HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![port], + allow_private_ips: true, + ..HttpConfig::default() + } +} + +fn spawn_test_server() -> (u16, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("test listener should bind"); + let port = listener + .local_addr() + .expect("test listener should have an address") + .port(); + let handle = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("test request should arrive"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream + .read(&mut buffer) + .expect("request should be readable"); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + assert!(request.starts_with(b"GET / HTTP/1.1")); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nX-Test: yes\r\n\r\nok") + .expect("response should be writable"); + }); + (port, handle) +} + +fn spawn_redirect_server( + status: u16, + redirects: usize, +) -> (u16, mpsc::Receiver, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("redirect listener should bind"); + let port = listener + .local_addr() + .expect("redirect listener should have an address") + .port(); + let (sender, receiver) = mpsc::channel(); + let handle = thread::spawn(move || { + for index in 0..=redirects { + let (mut stream, _) = listener.accept().expect("redirect request should arrive"); + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream + .read_exact(&mut byte) + .expect("redirect request headers should be readable"); + request.push(byte[0]); + } + let head = String::from_utf8(request).expect("request should be valid UTF-8"); + let content_length = head + .lines() + .find_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().expect("valid content length")) + }) + }) + .unwrap_or(0); + let mut body = vec![0; content_length]; + stream + .read_exact(&mut body) + .expect("redirect request body should be readable"); + sender + .send(format!("{head}{}", String::from_utf8_lossy(&body))) + .expect("request should be recorded"); + if index < redirects { + let location = if index + 1 == redirects { + format!("http://127.0.0.1:{port}/final") + } else { + format!("http://127.0.0.1:{port}/hop/{index}") + }; + write!( + stream, + "HTTP/1.1 {status} Redirect\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n" + ) + .expect("redirect response should be writable"); + } else { + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .expect("final response should be writable"); + } + } + }); + (port, receiver, handle) +} + +fn response_field<'a>(value: &'a Value, key: &str) -> &'a Value { + let Value::Map(map) = value else { + panic!("expected response map, got {value:?}"); + }; + map.get(&Value::string(key)) + .unwrap_or_else(|| panic!("response missing field {key}")) +} + +async fn drive_vm_to_halt(vm: &mut Vm) -> Result<(), vm::VmError> { + let mut status = vm.run()?; + loop { + match status { + VmStatus::Halted => return Ok(()), + VmStatus::Yielded => status = vm.resume()?, + VmStatus::Waiting(_) => { + vm.await_waiting_host_op().await?; + status = vm.resume()?; + } + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn http_host_executes_a_bounded_request_and_returns_a_response_map() { + let (port, server) = spawn_test_server(); + let mut vm = Vm::new(build_request_program(format!("http://127.0.0.1:{port}/"))); + vm.configure_http(local_http_config(port)) + .expect("HTTP configuration should be valid"); + install_host_driver(&mut vm); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + + drive_vm_to_halt(&mut vm) + .await + .expect("http request should complete"); + server.join().expect("test server should finish"); + + assert_eq!(response_field(&vm.stack()[0], "status"), &Value::Int(200)); + assert_eq!( + response_field(&vm.stack()[0], "body"), + &Value::bytes(b"ok".to_vec()) + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn buffered_redirect_rewrites_only_post_for_301_and_302() { + for status in [301, 302] { + for method in ["POST", "PUT", "PATCH", "DELETE", "OPTIONS"] { + let (port, requests, server) = spawn_redirect_server(status, 1); + let mut vm = Vm::new(build_request_program_with_method( + &format!("http://127.0.0.1:{port}/start"), + method, + )); + vm.configure_http(local_http_config(port)) + .expect("HTTP configuration should be valid"); + install_host_driver(&mut vm); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + + drive_vm_to_halt(&mut vm) + .await + .expect("redirected request should complete"); + assert_eq!(response_field(&vm.stack()[0], "status"), &Value::Int(200)); + + let first = requests.recv().expect("initial request should be recorded"); + let second = requests + .recv() + .expect("redirected request should be recorded"); + assert!( + first + .to_ascii_lowercase() + .starts_with(&format!("{} /start http/1.1", method.to_ascii_lowercase())), + "status {status}, method {method}: {first}" + ); + let expected_method = if method == "POST" { "GET" } else { method }; + assert!( + second.to_ascii_lowercase().starts_with(&format!( + "{} /final http/1.1", + expected_method.to_ascii_lowercase() + )), + "status {status}, method {method}: {second}" + ); + if method == "POST" { + assert!(!second.ends_with("payload"), "status {status}: {second}"); + } else { + assert!(second.ends_with("payload"), "status {status}: {second}"); + } + server.join().expect("redirect server should finish"); + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn buffered_redirect_chain_reaches_final_body() { + let (port, requests, server) = spawn_redirect_server(307, 2); + let mut vm = Vm::new(build_request_program(format!( + "http://127.0.0.1:{port}/start" + ))); + vm.configure_http(local_http_config(port)) + .expect("HTTP configuration should be valid"); + install_host_driver(&mut vm); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + + drive_vm_to_halt(&mut vm) + .await + .expect("redirect chain should complete"); + assert_eq!(response_field(&vm.stack()[0], "status"), &Value::Int(200)); + assert_eq!( + response_field(&vm.stack()[0], "body"), + &Value::bytes(b"ok".to_vec()) + ); + for _ in 0..3 { + requests + .recv() + .expect("each redirect request should be recorded"); + } + server.join().expect("redirect server should finish"); +} + +#[test] +fn http_host_rejects_targets_until_an_explicit_policy_allows_them() { + let mut vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + let error = vm + .run() + .expect_err("unconfigured HTTP targets must be rejected"); + assert!( + error.to_string().contains("HTTP host is not configured") + || error + .to_string() + .contains("HTTP target host is not allowed"), + "unexpected error: {error}" + ); +} + +#[test] +fn empty_registry_keeps_language_builtins_but_rejects_http_capability() { + let mut language_vm = Vm::new( + vm::compile_source("assert(true);") + .expect("language builtin program should compile") + .program, + ); + HostFunctionRegistry::empty() + .bind_vm_cached(&mut language_vm) + .expect("empty registry should bind a program without host imports"); + assert_eq!( + language_vm.run().expect("language builtin should run"), + VmStatus::Halted + ); + + let mut http_vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + let error = HostFunctionRegistry::restricted() + .bind_vm_cached(&mut http_vm) + .expect_err("unapproved HTTP capability must fail during preflight"); + assert!(error.to_string().contains("http::client::request")); +} + +#[test] +fn restricted_registry_requires_explicit_namespaced_builtin_capability() { + let compiled = compile_source( + r#"use io; +io::open("/tmp/rustscript-capability-test", "r");"#, + ) + .expect("namespaced host builtin should compile"); + let mut vm = Vm::new(compiled.program); + let error = HostFunctionRegistry::restricted() + .bind_vm_cached(&mut vm) + .expect_err("ungranted namespaced builtin must fail during preflight"); + assert!(error.to_string().contains("io_open")); +} + +#[test] +fn capability_binding_plan_cannot_cross_registry_profiles() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let unrestricted = HostFunctionRegistry::new(); + let plan = unrestricted + .prepare_plan(&program.imports) + .expect("unrestricted registry should prepare HTTP plan"); + let mut vm = Vm::new(program); + let error = HostFunctionRegistry::restricted() + .bind_vm_with_plan(&mut vm, &plan) + .expect_err("capability plan must not cross registry profiles"); + assert!(error.to_string().contains("different capability profile")); +} + +#[test] +fn capability_binding_plan_cannot_outlive_registry_mutation() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let mut registry = HostFunctionRegistry::new(); + let plan = registry + .prepare_plan(&program.imports) + .expect("registry should prepare HTTP plan"); + registry + .allow_builtin("http::client::request") + .expect("HTTP capability should be a known host callable"); + let mut vm = Vm::new(program); + let error = registry + .bind_vm_with_plan(&mut vm, &plan) + .expect_err("stale capability plan must not bind"); + assert!(error.to_string().contains("different capability profile")); +} + +#[test] +fn capability_binding_plan_detects_divergent_registry_clone_mutations() { + let unchanged_program = build_request_program("http://127.0.0.1:1/".to_string()); + let mut unchanged_registry = HostFunctionRegistry::restricted(); + unchanged_registry + .allow_builtin("http::client::request") + .expect("HTTP capability should be known"); + let unchanged_plan = unchanged_registry + .prepare_plan(&unchanged_program.imports) + .expect("restricted registry should prepare HTTP plan"); + let unchanged_clone = unchanged_registry.clone(); + let mut unchanged_vm = Vm::new(unchanged_program); + unchanged_clone + .bind_vm_with_plan(&mut unchanged_vm, &unchanged_plan) + .expect("an unchanged registry clone should reuse the plan"); + + let branch_program = build_request_program("http://127.0.0.1:1/".to_string()); + let branch_registry = HostFunctionRegistry::restricted(); + let mut first_mutation = branch_registry.clone(); + let mut second_mutation = branch_registry; + first_mutation + .allow_builtin("http::client::request") + .expect("HTTP capability should be known"); + second_mutation + .allow_builtin("io::open") + .expect("io capability should be known"); + let plan = first_mutation + .prepare_plan(&branch_program.imports) + .expect("first capability branch should prepare HTTP plan"); + let mut mutated_vm = Vm::new(branch_program); + let error = second_mutation + .bind_vm_with_plan(&mut mutated_vm, &plan) + .expect_err("divergent capability branches must reject each other's plan"); + assert!(error.to_string().contains("different capability profile")); +} + +#[test] +fn registry_state_rejects_structural_sibling_mutations() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let registry = HostFunctionRegistry::new(); + let mut source = registry.clone(); + let destination = registry; + source.register_static_args("test::structural", 0, |_args| { + Ok(CallOutcome::Return(CallReturn::One(Value::Null))) + }); + let plan = source + .prepare_plan(&program.imports) + .expect("mutated source registry should prepare HTTP plan"); + let mut vm = Vm::new(program); + let error = destination + .bind_vm_with_plan(&mut vm, &plan) + .expect_err("structural sibling mutation must reject the plan"); + assert!(error.to_string().contains("different registry state")); +} + +#[test] +fn cached_plan_refreshes_after_a_sibling_registry_mutation() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let registry = HostFunctionRegistry::new(); + let mut mutating_sibling = registry.clone(); + let destination = registry; + + let mut priming_vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + destination + .bind_vm_cached(&mut priming_vm) + .expect("destination should prime its plan cache"); + mutating_sibling.register_static_args("test::cache_refresh", 0, |_args| { + Ok(CallOutcome::Return(CallReturn::One(Value::Null))) + }); + + let mut refreshed_vm = Vm::new(program); + destination + .bind_vm_cached(&mut refreshed_vm) + .expect("destination should rebuild a plan after sibling mutation"); +} + +#[tokio::test(flavor = "current_thread")] +async fn max_stream_duration_does_not_shorten_buffered_requests() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(socket.read(&mut request).unwrap() > 0); + thread::sleep(std::time::Duration::from_millis(30)); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .unwrap(); + }); + let mut vm = Vm::new(build_request_program(format!("http://127.0.0.1:{port}/"))); + let mut buffered_config = local_http_config(port); + buffered_config.max_stream_duration = std::time::Duration::from_millis(1); + buffered_config.request_timeout = std::time::Duration::from_millis(200); + vm.configure_http(buffered_config).unwrap(); + install_host_driver(&mut vm); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + drive_vm_to_halt(&mut vm).await.unwrap(); + assert_eq!(response_field(&vm.stack()[0], "status"), &Value::Int(200)); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn explicitly_allowed_http_capability_reaches_http_policy() { + let mut vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + vm.configure_http(HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![1], + allow_private_ips: true, + ..HttpConfig::default() + }) + .expect("HTTP configuration should be valid"); + install_host_driver(&mut vm); + let mut registry = HostFunctionRegistry::restricted(); + registry + .allow_builtin("http::client::request") + .expect("HTTP builtin should be explicitly allowlisted"); + registry + .bind_vm_cached(&mut vm) + .expect("explicit capability plan should bind"); + let error = drive_vm_to_halt(&mut vm) + .await + .expect_err("connection failure should reach HTTP runtime"); + assert!(!matches!(error, vm::VmError::UnboundImport(_))); +} + +#[test] +fn http_in_flight_limit_rejects_before_starting_a_request() { + let mut vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + vm.set_http_max_in_flight(0); + vm.configure_http(HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![1], + allow_private_ips: true, + + ..HttpConfig::default() + }) + .expect("HTTP configuration should be valid"); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + let error = vm + .run() + .expect_err("zero in-flight capacity must reject the request"); + assert!(error.to_string().contains("in-flight request limit")); +} + +#[test] +fn http_config_accepts_bounded_stream_defaults_and_rejects_zero_bounds() { + let defaults = HttpConfig::default(); + defaults + .validate() + .expect("default HTTP stream bounds should be valid"); + assert!(defaults.max_stream_item_bytes > 0); + assert!(defaults.max_stream_total_bytes > 0); + assert!(defaults.max_sse_line_bytes > 0); + assert_eq!( + defaults.max_stream_duration, + std::time::Duration::from_secs(5 * 60) + ); + assert!(!defaults.stream_idle_timeout.is_zero()); + + HttpConfig { + max_stream_duration: std::time::Duration::from_millis(1), + ..defaults.clone() + } + .validate() + .expect("an explicit positive stream duration should be valid"); + + let invalid = [ + HttpConfig { + max_stream_item_bytes: 0, + ..defaults.clone() + }, + HttpConfig { + max_stream_total_bytes: 0, + ..defaults.clone() + }, + HttpConfig { + max_sse_line_bytes: 0, + ..defaults.clone() + }, + HttpConfig { + max_stream_duration: std::time::Duration::ZERO, + ..defaults.clone() + }, + HttpConfig { + stream_idle_timeout: std::time::Duration::ZERO, + ..defaults.clone() + }, + ]; + for config in invalid { + assert!(config.validate().is_err(), "zero stream bound must fail"); + } + + let mut vm = Vm::new(Program::new(Vec::new(), Vec::new())); + let error = vm + .configure_http(HttpConfig { + max_stream_item_bytes: 0, + ..HttpConfig::default() + }) + .expect_err("configuration must reject a zero stream bound"); + assert!(error.to_string().contains("max_stream_item_bytes")); + assert!(!vm.http_is_configured()); +} + +#[test] +fn http_config_rejects_request_timeout_that_cannot_form_a_deadline() { + let invalid = HttpConfig { + request_timeout: std::time::Duration::MAX, + ..HttpConfig::default() + }; + let validation_error = invalid + .validate() + .expect_err("overflowing request timeout must be rejected"); + assert!(validation_error.to_string().contains("request_timeout")); + + let mut vm = Vm::new(Program::new(Vec::new(), Vec::new())); + let configure_error = vm + .configure_http(invalid) + .expect_err("configuration must reject an overflowing request timeout"); + assert!(configure_error.to_string().contains("request_timeout")); + assert!(!vm.http_is_configured()); + + let invalid = HttpConfig { + max_stream_duration: std::time::Duration::MAX, + ..HttpConfig::default() + }; + let validation_error = invalid + .validate() + .expect_err("overflowing stream duration must be rejected"); + assert!(validation_error.to_string().contains("max_stream_duration")); + + let mut vm = Vm::new(Program::new(Vec::new(), Vec::new())); + let configure_error = vm + .configure_http(invalid) + .expect_err("configuration must reject an overflowing stream duration"); + assert!(configure_error.to_string().contains("max_stream_duration")); + assert!(!vm.http_is_configured()); +} + +#[derive(Default)] +struct RetirementState { + submitted: HashMap, + retired: Vec, +} + +struct RetirementBridge { + state: Arc>, +} + +impl HostAsyncBridge for RetirementBridge { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + self.state + .lock() + .expect("retirement state lock") + .submitted + .insert(op_id, future); + Ok(()) + } + + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown external host operation {op_id}" + )))) + } + + fn cancel_op(&mut self, op_id: HostOpId) { + let mut state = self.state.lock().expect("retirement state lock"); + state.submitted.remove(&op_id); + state.retired.push(op_id); + } +} + +fn pending_http_vm(state: Arc>) -> Vm { + let mut vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + vm.set_http_max_in_flight(1); + vm.configure_http(local_http_config(1)) + .expect("HTTP configuration should be valid"); + vm.set_async_bridge(Box::new(RetirementBridge { state })); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + assert!(matches!(vm.run(), Ok(VmStatus::Waiting(_)))); + vm +} + +#[test] +fn reset_retires_buffered_http_future_and_releases_its_permit() { + let state = Arc::new(Mutex::new(RetirementState::default())); + let mut vm = pending_http_vm(Arc::clone(&state)); + + vm.reset_for_reuse(); + + let retired_id = { + let state = state.lock().expect("retirement state lock"); + assert_eq!(state.submitted.len(), 0); + assert_eq!(state.retired.len(), 1); + state.retired[0] + }; + assert!( + vm.complete_host_op(retired_id, CallReturn::none()).is_err(), + "a retired future must not complete back into the VM" + ); + vm.configure_http(local_http_config(1)) + .expect("HTTP policy should remain reusable after reset"); + assert!( + matches!(vm.run(), Ok(VmStatus::Waiting(_))), + "a second request should acquire the released permit" + ); +} + +#[test] +fn shutdown_and_drop_retire_buffered_http_futures() { + let shutdown_state = Arc::new(Mutex::new(RetirementState::default())); + let mut vm = pending_http_vm(Arc::clone(&shutdown_state)); + vm.shutdown(); + { + let state = shutdown_state.lock().expect("retirement state lock"); + assert!(state.submitted.is_empty()); + assert_eq!(state.retired.len(), 1); + } + + let drop_state = Arc::new(Mutex::new(RetirementState::default())); + drop(pending_http_vm(Arc::clone(&drop_state))); + let state = drop_state.lock().expect("retirement state lock"); + assert!(state.submitted.is_empty()); + assert_eq!(state.retired.len(), 1); +} diff --git a/tests/vm/http_sse_tests.rs b/tests/vm/http_sse_tests.rs new file mode 100644 index 00000000..8a822ada --- /dev/null +++ b/tests/vm/http_sse_tests.rs @@ -0,0 +1,1048 @@ +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::mpsc; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use std::task::{Context, Poll}; +use std::thread; + +use vm::{ + CallOutcome, CallReturn, HostAsyncBridge, HostFunctionRegistry, HostFuture, HostFutureOutput, + HostOpId, HostStackFunction, HttpConfig, HttpHostExt, Value, Vm, VmError, VmMap, VmResult, + VmStatus, compile_source, +}; + +#[derive(Default)] +struct TokioHostDriver { + submitted: HashMap, +} + +impl HostAsyncBridge for TokioHostDriver { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + self.submitted.insert(op_id, future); + Ok(()) + } + + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown external host operation {op_id}" + )))) + } + + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + cx: &mut Context<'_>, + ) -> Poll> { + self.submitted.get_mut(&op_id).map_or_else( + || { + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))) + }, + |future| future.as_mut().poll(cx), + ) + } + + fn cancel_op(&mut self, op_id: HostOpId) { + self.submitted.remove(&op_id); + } +} + +struct AsyncWaitOnce { + calls: Arc, +} + +struct CountCalls { + calls: Arc, +} + +impl HostStackFunction for CountCalls { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> VmResult { + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(CallOutcome::Return(CallReturn::one(Value::Bool(true)))) + } +} + +impl HostStackFunction for AsyncWaitOnce { + fn call(&mut self, vm: &mut Vm, _args: &[Value]) -> VmResult { + if self.calls.fetch_add(1, Ordering::SeqCst) == 0 { + vm.submit_host_future(Box::pin(async move { + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + Ok(HostFutureOutput::returning(CallReturn::one(Value::Bool( + true, + )))) + })) + } else { + Ok(CallOutcome::Return(CallReturn::one(Value::Bool(true)))) + } + } +} + +fn field<'a>(value: &'a Value, key: &str) -> &'a Value { + let Value::Map(map) = value else { + panic!("expected map, got {value:?}"); + }; + map.get(&Value::string(key)) + .unwrap_or_else(|| panic!("missing field {key}")) +} + +fn map(entries: impl IntoIterator) -> Value { + Value::Map(Arc::new(VmMap::from_entries( + entries + .into_iter() + .map(|(key, value)| (Value::string(key), value)) + .collect(), + ))) +} + +async fn drive(vm: &mut Vm) -> VmResult<()> { + let mut status = vm.run()?; + loop { + match status { + VmStatus::Halted => return Ok(()), + VmStatus::Yielded => status = vm.resume()?, + VmStatus::Waiting(_) => { + vm.await_waiting_host_op().await?; + status = vm.resume()?; + } + } + } +} + +async fn run_sse_source(source: &str, config: HttpConfig) -> Result { + let compiled = compile_source(source).expect("SSE source should compile"); + let mut vm = Vm::new(compiled.program); + vm.configure_http(config).unwrap(); + vm.set_async_bridge(Box::::default()); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + drive(&mut vm).await.map(|()| vm) +} + +fn server(response_parts: Vec<&'static [u8]>) -> (u16, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let handle = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + let read = stream.read(&mut request).unwrap(); + let request = String::from_utf8_lossy(&request[..read]).to_ascii_lowercase(); + assert!(request.starts_with("get /events http/1.1")); + assert!(request.contains("accept: text/event-stream")); + for part in response_parts { + stream.write_all(part).unwrap(); + stream.flush().unwrap(); + } + }); + (port, handle) +} + +fn recording_server( + responses: Vec>, +) -> (u16, mpsc::Receiver, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let (sender, receiver) = mpsc::channel(); + let handle = thread::spawn(move || { + for response_parts in responses { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + } + let head = String::from_utf8(request).unwrap(); + let content_length = head + .lines() + .find_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + }) + .unwrap_or(0); + let mut body = vec![0; content_length]; + stream.read_exact(&mut body).unwrap(); + sender + .send(format!("{head}{}", String::from_utf8_lossy(&body))) + .unwrap(); + for part in response_parts { + stream.write_all(part).unwrap(); + stream.flush().unwrap(); + } + } + }); + (port, receiver, handle) +} + +fn config(port: u16) -> HttpConfig { + HttpConfig { + allowed_schemes: vec!["http".into()], + allowed_hosts: vec!["127.0.0.1".into()], + allowed_ports: vec![port], + allow_private_ips: true, + ..HttpConfig::default() + } +} + +fn assert_no_connection(listener: TcpListener, context: &'static str) -> thread::JoinHandle<()> { + thread::spawn(move || { + listener.set_nonblocking(true).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(500); + loop { + match listener.accept() { + Ok(_) => panic!("{context} must be rejected before a second connection"), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if std::time::Instant::now() >= deadline { + return; + } + thread::sleep(std::time::Duration::from_millis(5)); + } + Err(error) => panic!("unexpected accept error: {error}"), + } + } + }) +} + +fn rejecting_redirect_server( + location: impl FnOnce(u16) -> String + Send + 'static, +) -> (u16, mpsc::Receiver, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let location = location(port); + let (sender, receiver) = mpsc::channel(); + let handle = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + } + sender.send(String::from_utf8(request).unwrap()).unwrap(); + write!( + stream, + "HTTP/1.1 307 Temporary Redirect\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n" + ) + .unwrap(); + drop(stream); + listener.set_nonblocking(true).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(500); + loop { + match listener.accept() { + Ok(_) => panic!("invalid redirect must be rejected before a second connection"), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if std::time::Instant::now() >= deadline { + return; + } + thread::sleep(std::time::Duration::from_millis(5)); + } + Err(error) => panic!("unexpected accept error: {error}"), + } + } + }); + (port, receiver, handle) +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_delivers_open_events_end_and_terminal_summary() { + let (port, server) = server(vec![ + b"HTTP/1.1 200 OK\r\nContent-Type: Text/Event-Stream; charset=utf-8\r\nTransfer-Encoding: chunked\r\n\r\n", + b"b\r\ndata: one\n\n\r\n", + b"18\r\nevent: named\ndata: two\n\n\r\n", + b"0\r\n\r\n", + ]); + let source = format!( + r#" + use http; + fn record(item: map) -> map {{ + if item["kind"] == "open" && item["status"] != 200 {{ let _ = 1 / 0; }} + if item["kind"] == "event" && item["data"] == "one" && item["event"] != null {{ let _ = 1 / 0; }} + if item["kind"] == "event" && item["data"] == "two" && item["event"] != "named" {{ let _ = 1 / 0; }} + if item["kind"] == "end" && item != {{kind: "end"}} {{ let _ = 1 / 0; }} + {{action: "continue"}} + }} + let result = http::client::sse( + {{"method": "GET", "url": "http://127.0.0.1:{port}/events"}}, + record + ); + result; + "# + ); + let compiled = compile_source(&source).expect("SSE source should compile"); + let mut vm = Vm::new(compiled.program); + vm.configure_http(config(port)).unwrap(); + vm.set_async_bridge(Box::::default()); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + + drive(&mut vm).await.unwrap(); + server.join().unwrap(); + + let result = &vm.stack()[0]; + assert_eq!(field(result, "outcome"), &Value::string("eof")); + assert_eq!(field(result, "status"), &Value::Int(200)); + assert_eq!(field(result, "items"), &Value::Int(4)); + assert_eq!(field(result, "bytes_sent"), &Value::Int(0)); +} + +#[test] +fn sse_rejects_wrong_callback_schema_and_invalid_timeout_before_permit_admission() { + assert!(compile_source( + r#"use http; http::client::sse({"method":"GET","url":"http://127.0.0.1:1/"}, |item| 1);"# + ) + .is_err()); + + for (timeout, expected) in [ + ("0", "positive"), + ("-1", "positive"), + ("\"1\"", "type mismatch"), + ] { + let source = format!( + r#" + use http; + fn callback(item: map) -> map {{ {{action: "continue"}} }} + http::client::sse( + {{method: "GET", url: "http://127.0.0.1:1/events", timeout_ms: {timeout}}}, + callback + ); + "# + ); + let compiled = compile_source(&source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.set_http_max_in_flight(0); + vm.configure_http(config(1)).unwrap(); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + let error = vm.run().unwrap_err(); + assert!(error.to_string().contains(expected), "{timeout}: {error}"); + assert!( + !error.to_string().contains("in-flight request limit"), + "timeout validation must precede permit admission: {error}" + ); + } + + let source = r#" + use http; + fn callback(item: map) -> map { {action: "continue"} } + http::client::sse( + {method: "GET", url: "http://127.0.0.1:1/events", timeout_ms: 1}, + callback + ); + "#; + let compiled = compile_source(source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.set_http_max_in_flight(0); + vm.configure_http(config(1)).unwrap(); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + let error = vm.run().unwrap_err(); + assert!( + error.to_string().contains("in-flight request limit"), + "a positive timeout should pass timeout admission: {error}" + ); + + let source = r#" + use http; + fn callback(item: map) -> map { {action: "continue"} } + http::client::sse( + {method: "PUT", url: "http://127.0.0.1:1/events"}, + callback + ); + "#; + let compiled = compile_source(source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.configure_http(config(1)).unwrap(); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + let error = vm.run().unwrap_err(); + assert!(error.to_string().contains("GET or POST"), "{error}"); +} + +#[test] +fn sse_admission_does_not_require_a_tokio_reactor() { + let source = r#" + use http; + fn callback(item: map) -> map { {action: "continue"} } + http::client::sse( + {method: "GET", url: "http://127.0.0.1:1/events"}, + callback + ); + "#; + let compiled = compile_source(source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.configure_http(config(1)).unwrap(); + vm.set_async_bridge(Box::::default()); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + vm.reset_for_reuse(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_accepts_post_with_body() { + let (port, requests, server) = recording_server(vec![vec![ + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n", + ]]); + let source = format!( + r#" + use http; + fn callback(item: map) -> map {{ {{action: "continue"}} }} + http::client::sse( + {{method: "POST", url: "http://127.0.0.1:{port}/events", body: "payload"}}, + callback + ); + "# + ); + let vm = run_sse_source(&source, config(port)).await.unwrap(); + assert_eq!(field(&vm.stack()[0], "outcome"), &Value::string("eof")); + let request = requests.recv().unwrap().to_ascii_lowercase(); + assert!(request.starts_with("post /events http/1.1")); + assert!(request.ends_with("payload")); + server.join().unwrap(); +} + +fn redirect_server(status: u16) -> (u16, mpsc::Receiver, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let (sender, receiver) = mpsc::channel(); + let handle = thread::spawn(move || { + for index in 0..2 { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + } + let head = String::from_utf8(request).unwrap(); + let length = head + .lines() + .find_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + }) + .unwrap_or(0); + let mut body = vec![0; length]; + stream.read_exact(&mut body).unwrap(); + sender + .send(format!("{head}{}", String::from_utf8_lossy(&body))) + .unwrap(); + if index == 0 { + write!( + stream, + "HTTP/1.1 {status} Redirect\r\nLocation: http://127.0.0.1:{port}/final\r\nContent-Length: 0\r\n\r\n" + ) + .unwrap(); + } else { + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n") + .unwrap(); + } + } + }); + (port, receiver, handle) +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_post_redirect_method_and_body_follow_http_rules() { + for (status, preserves_post) in [ + (301, false), + (302, false), + (303, false), + (307, true), + (308, true), + ] { + let (port, requests, server) = redirect_server(status); + let source = format!( + r#"use http; + fn callback(item: map) -> map {{ {{action: "continue"}} }} + http::client::sse({{method:"POST", url:"http://127.0.0.1:{port}/start", body:"payload"}}, callback);"# + ); + run_sse_source(&source, config(port)).await.unwrap(); + let first = requests.recv().unwrap().to_ascii_lowercase(); + let second = requests.recv().unwrap().to_ascii_lowercase(); + assert!(first.starts_with("post /start http/1.1")); + if preserves_post { + assert!( + second.starts_with("post /final http/1.1"), + "status {status}: {second}" + ); + assert!(second.ends_with("payload"), "status {status}: {second}"); + } else { + assert!( + second.starts_with("get /final http/1.1"), + "status {status}: {second}" + ); + assert!(!second.ends_with("payload"), "status {status}: {second}"); + } + server.join().unwrap(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_get_redirect_preserves_get_for_301_and_302() { + for status in [301, 302] { + let (port, requests, server) = redirect_server(status); + let source = format!( + r#"use http; + fn callback(item: map) -> map {{ {{action: "continue"}} }} + http::client::sse({{method:"GET", url:"http://127.0.0.1:{port}/start"}}, callback);"# + ); + run_sse_source(&source, config(port)).await.unwrap(); + let first = requests.recv().unwrap().to_ascii_lowercase(); + let second = requests.recv().unwrap().to_ascii_lowercase(); + assert!(first.starts_with("get /start http/1.1")); + assert!( + second.starts_with("get /final http/1.1"), + "status {status}: {second}" + ); + assert!(!second.ends_with("payload"), "status {status}: {second}"); + server.join().unwrap(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_rejects_redirect_userinfo_before_reconnecting() { + let (port, requests, server) = rejecting_redirect_server(|port| { + format!("http://redirect-user:redirect-password@127.0.0.1:{port}/final") + }); + let source = format!( + r#"use http; + fn callback(item: map) -> map {{ {{action: "continue"}} }} + http::client::sse( + {{method:"GET", url:"http://127.0.0.1:{port}/start", headers:{{Authorization:"Bearer secret", Cookie:"a=b"}}}}, + callback + );"# + ); + let error = match run_sse_source(&source, config(port)).await { + Ok(_) => panic!("redirect userinfo must be rejected"), + Err(error) => error, + }; + assert!( + error.to_string().contains("URL userinfo is not allowed"), + "{error}" + ); + let request = requests.recv().unwrap().to_ascii_lowercase(); + assert!(request.contains("authorization:")); + assert!(request.contains("cookie: a=b")); + assert!(!request.contains("redirect-user")); + assert!(!request.contains("redirect-password")); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_rejects_disallowed_redirect_targets_before_connecting() { + for (host, allow_target_port, expected) in [ + ("127.0.0.1", false, "target port"), + ("localhost", true, "target host"), + ] { + let target_listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let target_port = target_listener.local_addr().unwrap().port(); + let no_target_connection = assert_no_connection(target_listener, expected); + let location = format!("http://{host}:{target_port}/final"); + let redirect = format!( + "HTTP/1.1 307 Temporary Redirect\r\nLocation: {location}\r\nContent-Length: 0\r\n\r\n" + ); + let redirect = Box::leak(redirect.into_bytes().into_boxed_slice()); + let (source_port, requests, source_server) = recording_server(vec![vec![redirect]]); + let source = format!( + r#"use http; + fn callback(item: map) -> map {{ {{action: "continue"}} }} + http::client::sse( + {{method:"GET", url:"http://127.0.0.1:{source_port}/start", headers:{{Authorization:"Bearer secret", Cookie:"a=b"}}}}, + callback + );"# + ); + let mut allowed = config(source_port); + if allow_target_port { + allowed.allowed_ports.push(target_port); + } + let error = match run_sse_source(&source, allowed).await { + Ok(_) => panic!("disallowed redirect target must be rejected"), + Err(error) => error, + }; + assert!(error.to_string().contains(expected), "{error}"); + let request = requests.recv().unwrap().to_ascii_lowercase(); + assert!(request.contains("authorization:")); + assert!(request.contains("cookie: a=b")); + source_server.join().unwrap(); + no_target_connection.join().unwrap(); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_stop_retires_without_end_and_returns_stopped_summary() { + let (port, server) = server(vec![ + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n", + b"9\r\ndata: x\n\n\r\n", + b"0\r\n\r\n", + ]); + let source = format!( + r#"use http; + fn stop(item: map) -> map {{ {{action: "stop"}} }} + http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, stop);"# + ); + let vm = run_sse_source(&source, config(port)).await.unwrap(); + server.join().unwrap(); + assert_eq!(field(&vm.stack()[0], "outcome"), &Value::string("stopped")); + assert_eq!(field(&vm.stack()[0], "items"), &Value::Int(1)); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_reset_releases_the_connection_permit_before_reuse() { + let (port, server) = server(vec![ + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n", + ]); + let source = format!( + r#"use http; + http::client::sse( + {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, + |item| {{action: "continue"}} + );"# + ); + let compiled = compile_source(&source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.set_http_max_in_flight(1); + vm.configure_http(config(port)).unwrap(); + vm.set_async_bridge(Box::::default()); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + + assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + vm.reset_for_reuse(); + drive(&mut vm).await.unwrap(); + assert_eq!(field(&vm.stack()[0], "outcome"), &Value::string("eof")); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_rejects_status_content_type_and_idle_peer() { + for (head, expected) in [ + (b"HTTP/1.1 404 Not Found\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n".as_slice(), "status 404"), + (b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 0\r\n\r\n".as_slice(), "Content-Type"), + ] { + let (port, server) = server(vec![head]); + let source = format!( + r#"use http; fn go(item: map) -> map {{ {{action:"continue"}} }} http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, go);"# + ); + let error = match run_sse_source(&source, config(port)).await { + Ok(_) => panic!("invalid SSE response must fail"), + Err(error) => error, + }; + assert!(error.to_string().contains(expected), "{error}"); + server.join().unwrap(); + } + + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + let read = socket.read(&mut request).unwrap(); + assert!(read > 0, "SSE request should be received"); + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n").unwrap(); + socket.flush().unwrap(); + thread::sleep(std::time::Duration::from_millis(80)); + }); + let mut idle_config = config(port); + idle_config.stream_idle_timeout = std::time::Duration::from_millis(20); + let source = format!( + r#"use http; fn go(item: map) -> map {{ {{action:"continue"}} }} http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, go);"# + ); + let error = match run_sse_source(&source, idle_config).await { + Ok(_) => panic!("idle SSE peer must time out"), + Err(error) => error, + }; + assert!(error.to_string().contains("idle timeout"), "{error}"); + server.join().unwrap(); + + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + let read = socket.read(&mut request).unwrap(); + assert!(read > 0, "SSE request should be received"); + thread::sleep(std::time::Duration::from_millis(80)); + }); + let mut opening_config = config(port); + opening_config.stream_idle_timeout = std::time::Duration::from_millis(20); + let source = format!( + r#"use http; fn go(item: map) -> map {{ {{action:"continue"}} }} http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, go);"# + ); + let error = match run_sse_source(&source, opening_config).await { + Ok(_) => panic!("SSE response opening must obey idle timeout"), + Err(error) => error, + }; + assert!( + error.to_string().contains("idle timeout while opening"), + "{error}" + ); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_script_timeout_shortens_the_host_stream_duration() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(socket.read(&mut request).unwrap() > 0); + thread::sleep(std::time::Duration::from_millis(80)); + }); + let mut deadline_config = config(port); + deadline_config.max_stream_duration = std::time::Duration::from_millis(200); + deadline_config.stream_idle_timeout = std::time::Duration::from_millis(200); + let source = format!( + r#"use http; fn go(item: map) -> map {{ {{action:"continue"}} }} http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events","timeout_ms":20}}, go);"# + ); + let error = match run_sse_source(&source, deadline_config).await { + Ok(_) => panic!("script deadline should shorten the host maximum"), + Err(error) => error, + }; + assert!(error.to_string().contains("total deadline"), "{error}"); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_host_stream_duration_caps_script_timeout_while_opening() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(socket.read(&mut request).unwrap() > 0); + thread::sleep(std::time::Duration::from_millis(80)); + }); + let mut deadline_config = config(port); + deadline_config.max_stream_duration = std::time::Duration::from_millis(20); + deadline_config.stream_idle_timeout = std::time::Duration::from_millis(200); + let source = format!( + r#"use http; fn go(item: map) -> map {{ {{action:"continue"}} }} http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events","timeout_ms":1000}}, go);"# + ); + let error = match run_sse_source(&source, deadline_config).await { + Ok(_) => panic!("host duration should cap the script timeout during opening"), + Err(error) => error, + }; + assert!(error.to_string().contains("total deadline"), "{error}"); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_total_deadline_expires_despite_periodic_progress_below_idle_timeout() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(socket.read(&mut request).unwrap() > 0); + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n").unwrap(); + socket.flush().unwrap(); + for _ in 0..40 { + thread::sleep(std::time::Duration::from_millis(25)); + if socket.write_all(b"c\r\ndata: tick\n\n\r\n").is_err() { + break; + } + if socket.flush().is_err() { + break; + } + } + }); + let mut deadline_config = config(port); + deadline_config.max_stream_duration = std::time::Duration::from_millis(600); + deadline_config.stream_idle_timeout = std::time::Duration::from_millis(250); + let callbacks = Arc::new(AtomicUsize::new(0)); + let source = format!( + r#"use http; + fn count_call() -> bool; + fn go(item: map) -> map {{ + {{action: if count_call() => {{"continue"}} else => {{"continue"}}}} + }} + http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, go);"# + ); + let compiled = compile_source(&source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.configure_http(deadline_config).unwrap(); + vm.set_async_bridge(Box::::default()); + let mut registry = HostFunctionRegistry::new(); + registry.register_stack("count_call", 0, { + let callbacks = Arc::clone(&callbacks); + move || { + Box::new(CountCalls { + calls: Arc::clone(&callbacks), + }) + } + }); + registry.bind_vm_cached(&mut vm).unwrap(); + let error = drive(&mut vm) + .await + .expect_err("periodic progress must not extend the total deadline"); + assert!(error.to_string().contains("total deadline"), "{error}"); + server.join().unwrap(); + assert!( + callbacks.load(Ordering::SeqCst) >= 4, + "multiple progress events must reach callbacks inside the idle bound" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_total_deadline_releases_the_connection_permit_for_reuse() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut first, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(first.read(&mut request).unwrap() > 0); + let first = thread::spawn(move || { + thread::sleep(std::time::Duration::from_millis(80)); + drop(first); + }); + + let (mut second, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(second.read(&mut request).unwrap() > 0); + second + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n", + ) + .unwrap(); + first.join().unwrap(); + }); + let source = format!( + r#"use http; http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, |item| {{action:"continue"}});"# + ); + let compiled = compile_source(&source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.set_http_max_in_flight(1); + let mut deadline_config = config(port); + deadline_config.max_stream_duration = std::time::Duration::from_millis(20); + deadline_config.stream_idle_timeout = std::time::Duration::from_millis(200); + vm.configure_http(deadline_config).unwrap(); + vm.set_async_bridge(Box::::default()); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + + let error = drive(&mut vm) + .await + .expect_err("the first stream should reach its total deadline"); + assert!(error.to_string().contains("total deadline"), "{error}"); + vm.reset_for_reuse(); + drive(&mut vm) + .await + .expect("the second stream should acquire the released permit"); + assert_eq!(field(&vm.stack()[0], "outcome"), &Value::string("eof")); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_callback_stop_after_deadline_fails_and_releases_permit_without_another_poll() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut first, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(first.read(&mut request).unwrap() > 0); + first + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n", + ) + .unwrap(); + first.flush().unwrap(); + let first = thread::spawn(move || { + thread::sleep(std::time::Duration::from_millis(500)); + drop(first); + }); + + let (mut second, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(second.read(&mut request).unwrap() > 0); + second + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: 0\r\n\r\n", + ) + .unwrap(); + first.join().unwrap(); + }); + let source = format!( + r#" + use http; + fn async_wait() -> bool; + http::client::sse( + {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, + |item| {{ + action: if async_wait() => {{ "stop" }} else => {{ "stop" }} + }} + ); + "# + ); + let compiled = compile_source(&source).unwrap(); + let mut vm = Vm::new(compiled.program); + vm.set_http_max_in_flight(1); + let mut deadline_config = config(port); + deadline_config.max_stream_duration = std::time::Duration::from_millis(100); + deadline_config.stream_idle_timeout = std::time::Duration::from_secs(1); + vm.configure_http(deadline_config).unwrap(); + vm.set_async_bridge(Box::::default()); + let wait_calls = Arc::new(AtomicUsize::new(0)); + let mut registry = HostFunctionRegistry::new(); + registry.register_stack("async_wait", 0, { + let wait_calls = Arc::clone(&wait_calls); + move || { + Box::new(AsyncWaitOnce { + calls: Arc::clone(&wait_calls), + }) + } + }); + registry.bind_vm_cached(&mut vm).unwrap(); + + let error = drive(&mut vm) + .await + .expect_err("a callback action after the total deadline must fail"); + assert!( + matches!(error, VmError::HostError(ref message) if message == "SSE total deadline exceeded"), + "{error}" + ); + assert_eq!(wait_calls.load(Ordering::SeqCst), 1); + assert!(vm.stack().iter().all(|value| { + let Value::Map(map) = value else { + return true; + }; + map.get(&Value::string("outcome")) != Some(&Value::string("stopped")) + })); + + vm.reset_for_reuse(); + drive(&mut vm) + .await + .expect("the next stream should acquire the released permit"); + assert_eq!(wait_calls.load(Ordering::SeqCst), 2); + assert_eq!(field(&vm.stack()[0], "outcome"), &Value::string("stopped")); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_callback_continue_after_deadline_fails_before_another_network_poll() { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + let mut request = [0; 1024]; + assert!(socket.read(&mut request).unwrap() > 0); + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\n\r\n", + ) + .unwrap(); + socket.flush().unwrap(); + thread::sleep(std::time::Duration::from_millis(500)); + }); + let source = format!( + r#" + use http; + fn async_wait() -> bool; + http::client::sse( + {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, + |item| {{ + action: if async_wait() => {{ "continue" }} else => {{ "continue" }} + }} + ); + "# + ); + let compiled = compile_source(&source).unwrap(); + let mut vm = Vm::new(compiled.program); + let mut deadline_config = config(port); + deadline_config.max_stream_duration = std::time::Duration::from_millis(100); + deadline_config.stream_idle_timeout = std::time::Duration::from_secs(1); + vm.configure_http(deadline_config).unwrap(); + vm.set_async_bridge(Box::::default()); + let wait_calls = Arc::new(AtomicUsize::new(0)); + let mut registry = HostFunctionRegistry::new(); + registry.register_stack("async_wait", 0, { + let wait_calls = Arc::clone(&wait_calls); + move || { + Box::new(AsyncWaitOnce { + calls: Arc::clone(&wait_calls), + }) + } + }); + registry.bind_vm_cached(&mut vm).unwrap(); + + let error = drive(&mut vm) + .await + .expect_err("a continue action after the total deadline must fail"); + assert!( + matches!(error, VmError::HostError(ref message) if message == "SSE total deadline exceeded"), + "{error}" + ); + assert_eq!(wait_calls.load(Ordering::SeqCst), 1); + server.join().unwrap(); +} + +#[tokio::test(flavor = "current_thread")] +async fn sse_revalidates_redirects_and_strips_cross_origin_credentials() { + let (target_port, target_requests, target) = recording_server(vec![vec![ + b"HTTP/1.1 200 OK\r\nContent-Type: Text/Event-Stream; Charset=UTF-8\r\nX-Obs: \x80\r\nContent-Length: 0\r\n\r\n", + ]]); + let redirect = format!( + "HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:{target_port}/final\r\nContent-Length: 0\r\n\r\n" + ); + let redirect = Box::leak(redirect.into_bytes().into_boxed_slice()); + let (source_port, source_requests, source_server) = recording_server(vec![vec![redirect]]); + let source_code = format!( + r#" + use http; + fn record(item: map) -> map {{ + if item["kind"] == "open" && item != {{ + kind: "open", + status: 200, + headers: {{"content-type": "Text/Event-Stream; Charset=UTF-8", "x-obs": b"\x80", "content-length": "0"}}, + url: "http://127.0.0.1:{target_port}/final" + }} {{ let _ = 1 / 0; }} + if item["kind"] == "end" && item != {{kind: "end"}} {{ let _ = 1 / 0; }} + {{action: "continue"}} + }} + http::client::sse( + {{method: "POST", url: "http://127.0.0.1:{source_port}/start", body: "payload", headers: {{Authorization: "Bearer secret", Cookie: "a=b"}}}}, + record + ); + "# + ); + let mut allowed = config(source_port); + allowed.allowed_ports.push(target_port); + let vm = run_sse_source(&source_code, allowed).await.unwrap(); + let final_url = format!("http://127.0.0.1:{target_port}/final"); + assert_eq!( + &vm.stack()[0], + &map([ + ("outcome", Value::string("eof")), + ("status", Value::Int(200)), + ( + "headers", + map([ + ( + "content-type", + Value::string("Text/Event-Stream; Charset=UTF-8"), + ), + ("x-obs", Value::bytes(vec![0x80])), + ("content-length", Value::string("0")), + ]), + ), + ("url", Value::string(final_url)), + ("items", Value::Int(2)), + ("bytes_received", Value::Int(0)), + ("bytes_sent", Value::Int(0)), + ]) + ); + let first = source_requests.recv().unwrap().to_ascii_lowercase(); + assert!(first.starts_with("post /start http/1.1")); + assert!(first.ends_with("payload")); + assert!(first.contains("authorization: bearer secret")); + assert!(first.contains("cookie: a=b")); + let second = target_requests.recv().unwrap().to_ascii_lowercase(); + assert!(second.starts_with("post /final http/1.1")); + assert!(second.ends_with("payload")); + assert!(!second.contains("authorization:")); + assert!(!second.contains("cookie:")); + source_server.join().unwrap(); + target.join().unwrap(); +}