From 69b42aa8d086bf31ba2c10e4c50cfca824c96ddd Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Thu, 27 Aug 2026 22:32:38 -0400 Subject: [PATCH] feat: add WebTransport sessions --- README.md | 2 + docs/cli-reference.md | 12 + docs/index.md | 1 + docs/webtransport.md | 25 ++ go.mod | 2 + go.sum | 4 + internal/cli/app.go | 83 ++++- internal/cli/cli.go | 89 ++++++ internal/cli/registry.go | 34 +- internal/cli/webtransport_test.go | 33 ++ internal/client/client.go | 284 +++++++++-------- internal/client/webtransport.go | 227 ++++++++++++++ internal/config/config.go | 16 +- internal/core/core.go | 30 ++ internal/core/limits.go | 2 + internal/fetch/fetch.go | 14 +- internal/fetch/grpc_reflection.go | 4 + internal/fetch/webtransport.go | 241 ++++++++++++++ internal/wt/wt.go | 396 ++++++++++++++++++++++++ internal/wt/wt_test.go | 100 ++++++ main.go | 19 +- skills/fetch/SKILL.md | 17 +- skills/fetch/evals/evals.json | 20 ++ skills/fetch/references/webtransport.md | 18 ++ 24 files changed, 1531 insertions(+), 142 deletions(-) create mode 100644 docs/webtransport.md create mode 100644 internal/cli/webtransport_test.go create mode 100644 internal/client/webtransport.go create mode 100644 internal/fetch/webtransport.go create mode 100644 internal/wt/wt.go create mode 100644 internal/wt/wt_test.go create mode 100644 skills/fetch/references/webtransport.md diff --git a/README.md b/README.md index 306780ea..faa0d6df 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ A modern HTTP(S) client for the command line, implemented in Go. - **Response formatting** - Automatic formatting and syntax highlighting for JSON, XML, YAML, HTML, CSS, CSV, Markdown, MessagePack, Protocol Buffers, and more - **Image rendering** - Display images directly in your terminal - **WebSocket support** - Bidirectional WebSocket connections with automatic JSON formatting +- **WebTransport support** - HTTP/3 stream and datagram sessions over direct UDP - **gRPC support** - Make gRPC calls with automatic reflection, discovery, and JSON-to-protobuf conversion - **Authentication** - Built-in support for Basic Auth, Bearer Token, AWS Signature V4, and mTLS - **Compression** - Select automatic, Brotli, gzip, zstd, or disabled response decoding @@ -58,6 +59,7 @@ fetch picsum.photos/1024/1024 - **[Output Formatting](docs/output-formatting.md)** - Supported content types and formatting options - **[Image Rendering](docs/image-rendering.md)** - Terminal image protocols and formats - **[WebSocket](docs/websocket.md)** - Bidirectional WebSocket connections +- **[WebTransport](docs/webtransport.md)** - HTTP/3 stream and datagram sessions - **[gRPC](docs/grpc.md)** - Making gRPC requests with Protocol Buffers - **[Advanced Features](docs/advanced-features.md)** - DNS, proxies, TLS, HTTP versions, and more - **[Encrypted ClientHello](docs/ech.md)** - ECH modes, discovery, and downgrade safety diff --git a/docs/cli-reference.md b/docs/cli-reference.md index a414c7cc..4c8bd3ae 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -354,6 +354,18 @@ valid only with a `ws://` or `wss://` URL. Text lines, interactive entries, and incoming messages are bounded to 16 MiB; binary stdin is streamed in bounded chunks. See [WebSocket](websocket.md). +### WebTransport options + +`--webtransport URL` opens an HTTPS WebTransport session over HTTP/3 and direct +UDP. The default `--wt-mode stream` uses one reliable bidirectional stream. +`--wt-mode datagram` uses unreliable datagrams. Use `--wt-datagram-mode +lines|binary` to split piped input, and repeat `--wt-protocol PROTOCOL` to offer +application protocols. Received datagrams are compact JSON Lines records with +base64 data. WebTransport does not support proxies, redirects, retries, +formatting, HAR, output files, Unix sockets, or Digest authentication. EOF on +datagram input does not close the session; use Ctrl+C when the peer remains +open. `--dry-run` does not access the network or consume stdin. + ## Agent Skill Options ### `--skill` diff --git a/docs/index.md b/docs/index.md index f07c764b..43e0cf7a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,6 +20,7 @@ reference by task. - [DNS, proxy, HTTP versions, and TLS/ECH](advanced-features.md) - [Encrypted ClientHello](ech.md) - [WebSockets](websocket.md) +- [WebTransport](webtransport.md) - [gRPC](grpc.md) - [Image rendering](image-rendering.md) - [Self-update and installation](updates.md) diff --git a/docs/webtransport.md b/docs/webtransport.md new file mode 100644 index 00000000..95c16e75 --- /dev/null +++ b/docs/webtransport.md @@ -0,0 +1,25 @@ +# WebTransport + +Use `fetch --webtransport https://host/path` to open a WebTransport session. +WebTransport uses HTTP/3 over a direct UDP connection. It does not support +proxies, Unix sockets, redirects, retries, output files, formatting, or +Digest authentication. + +The default mode is one reliable bidirectional stream. `-d` and `-j` are sent +after the session handshake, followed by piped standard input. The stream is +closed for writing at EOF, but fetch continues to read until the peer closes. +Stream output is raw bytes when stdout is redirected and is escaped when it is +a terminal. + +Use `--wt-mode datagram` for unreliable datagrams. `--wt-datagram-mode lines` +sends one datagram per line; `binary` sends 1 KiB chunks. Received datagrams +are JSON Lines records with `sequence`, `length`, and base64 `data` fields. +Datagram input ending does not close the session. Use Ctrl+C when the peer does +not close it. + +Repeat `--wt-protocol` to advertise application protocols. Protocols are +validated and sent in offer order. `--dry-run` prints the CONNECT metadata and +does not resolve DNS, open UDP, or consume standard input. + +This implementation uses WebTransport draft-16. Use HTTPS and TLS 1.3-capable +HTTP/3 servers. diff --git a/go.mod b/go.mod index a4cf73af..a13685f1 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,12 @@ go 1.27.0 require ( github.com/andybalholm/brotli v1.2.2 github.com/coder/websocket v1.8.15 + github.com/dunglas/httpsfv v1.1.0 github.com/goccy/go-yaml v1.19.2 github.com/klauspost/compress v1.19.2 github.com/mattn/go-runewidth v0.0.28 github.com/quic-go/quic-go v0.61.0 + github.com/quic-go/webtransport-go v0.12.0 github.com/ryanfowler/readability v0.1.1 github.com/tinylib/msgp v1.6.4 github.com/yuin/goldmark v1.8.5 diff --git a/go.sum b/go.sum index 60ce3085..e56f5f59 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNU github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dunglas/httpsfv v1.1.0 h1:Jw76nAyKWKZKFrpMMcL76y35tOpYHqQPzHQiwDvpe54= +github.com/dunglas/httpsfv v1.1.0/go.mod h1:zID2mqw9mFsnt7YC3vYQ9/cjq30q41W+1AnDwH8TiMg= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -24,6 +26,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.61.0 h1:ui88A53s8MSVYLC56en0KQ17HARk+9986Dn0SBfKNvA= github.com/quic-go/quic-go v0.61.0/go.mod h1:9So2anK4Tp22URSQq00k+Vo2PNkle96ycDPDHL4s9vs= +github.com/quic-go/webtransport-go v0.12.0 h1:CpnKNwZvdV0LD73xoHO8QaR0NI3llqpWRwnazdZS0sE= +github.com/quic-go/webtransport-go v0.12.0/go.mod h1:GHne8aRFJ24h73pAMrcywXtuaz/ShBXCLXLvG/NPFdU= github.com/ryanfowler/readability v0.1.1 h1:MpvDWXpeawWSvvUj6YTXL8vqWhz06FuKRvaap2NVrZc= github.com/ryanfowler/readability v0.1.1/go.mod h1:rNsnkYiYbZXCpEyIZyXbg26Dd0Z+6+11QIm6PBeeS1A= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= diff --git a/internal/cli/app.go b/internal/cli/app.go index 5472db49..546d2634 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -8,6 +8,7 @@ import ( "os" "strings" + "github.com/dunglas/httpsfv" "github.com/ryanfowler/fetch/internal/aws" "github.com/ryanfowler/fetch/internal/config" "github.com/ryanfowler/fetch/internal/core" @@ -52,7 +53,12 @@ type App struct { Force bool SortHeaders bool WS bool // set when URL scheme is ws:// or wss:// + WebTransport bool + WTMode core.WTMode + WTDgramMode core.WTDatagramMode + WTProtocols []string Method string + MethodExplicit bool Multipart []core.KeyVal[string] Output string ProtoDesc string @@ -78,6 +84,9 @@ type App struct { pagerSet bool noPagerSet bool wsMessageModeSet bool + wsInteractiveSet bool + wtDgramModeSet bool + wtModeSet bool provenance map[string]OptionProvenance } @@ -571,12 +580,13 @@ func (a *App) CLI() *CLI { }, boolFlag(&a.Version, "version", "V", "Print version"), + boolFlag(&a.WebTransport, "webtransport", "", "Use WebTransport over HTTP/3"), Flag{ Long: "ws-interactive", Args: "MODE", Description: "WebSocket prompt mode", - IsSet: func() bool { return a.WSInteractive != core.WSInteractiveAuto }, + IsSet: func() bool { return a.wsInteractiveSet }, Fn: a.parseWSInteractiveFlag, }.WithValues([]core.KeyVal[string]{ {Key: "auto", Val: "Use interactive prompt when attached to a terminal"}, @@ -599,6 +609,14 @@ func (a *App) CLI() *CLI { {Key: "binary", Val: "Send binary messages"}, }), + Flag{Long: "wt-datagram-mode", Args: "MODE", Description: "WT datagram stdin mode", IsSet: func() bool { return a.wtDgramModeSet }, Fn: a.parseWTDgramModeFlag}.WithValues([]core.KeyVal[string]{ + {Key: "lines", Val: "One datagram per line"}, {Key: "binary", Val: "One datagram per 1 KiB"}, + }), + Flag{Long: "wt-mode", Args: "MODE", Description: "WT data mode", IsSet: func() bool { return a.wtModeSet }, Fn: a.parseWTModeFlag}.WithValues([]core.KeyVal[string]{ + {Key: "stream", Val: "Reliable bidirectional stream"}, {Key: "datagram", Val: "Unreliable datagrams"}, + }), + {Long: "wt-protocol", Args: "PROTOCOL", Description: "WT application protocol (repeatable)", IsSet: func() bool { return len(a.WTProtocols) > 0 }, Fn: a.parseWTProtocolFlag}, + // Custom: XML body { Short: "x", @@ -676,9 +694,15 @@ func (a *App) parseDataFlag(value string) error { if err != nil { return err } - a.Data, a.ContentType, err = core.DetectContentType(r, path) - if err != nil { - return err + // Stdin is one-shot. Do not sniff it while parsing: WebTransport defers + // application input until after its handshake, and dry-run must not read it. + if value == "@-" && a.WebTransport { + a.Data, a.ContentType = r, "application/octet-stream" + } else { + a.Data, a.ContentType, err = core.DetectContentType(r, path) + if err != nil { + return err + } } a.dataSet = true return nil @@ -811,7 +835,58 @@ func (a *App) parseXMLFlag(value string) error { return nil } +func (a *App) parseWTModeFlag(value string) error { + switch value { + case "stream": + a.WTMode = core.WTStream + case "datagram": + a.WTMode = core.WTDatagram + default: + return core.NewValueError("wt-mode", value, "must be one of [stream, datagram]", false) + } + a.wtModeSet = true + return nil +} + +func validateWTProtocol(value string) error { + _, err := httpsfv.Marshal(httpsfv.NewItem(value)) + return err +} + +func (a *App) parseWTProtocolFlag(value string) error { + if strings.TrimSpace(value) == "" { + return core.NewValueError("wt-protocol", value, "must not be empty", false) + } + for _, protocol := range a.WTProtocols { + if protocol == value { + return core.NewValueError("wt-protocol", value, "duplicate protocol", false) + } + } + // Validate with the same Structured Fields encoder used by webtransport-go. + // Keep this dependency out of normal parsing by validating the item syntax + // through the small shared helper. + if err := validateWTProtocol(value); err != nil { + return core.NewValueError("wt-protocol", value, err.Error(), false) + } + a.WTProtocols = append(a.WTProtocols, value) + return nil +} + +func (a *App) parseWTDgramModeFlag(value string) error { + switch value { + case "lines": + a.WTDgramMode = core.WTDatagramLines + case "binary": + a.WTDgramMode = core.WTDatagramBinary + default: + return core.NewValueError("wt-datagram-mode", value, "must be one of [lines, binary]", false) + } + a.wtDgramModeSet = true + return nil +} + func (a *App) parseWSInteractiveFlag(value string) error { + a.wsInteractiveSet = true switch value { case "auto": a.WSInteractive = core.WSInteractiveAuto diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 462371ec..d2230cb4 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -2,6 +2,8 @@ package cli import ( "bytes" + "crypto/tls" + "errors" "fmt" "io" "net/url" @@ -12,6 +14,7 @@ import ( "strings" "time" + "github.com/ryanfowler/fetch/internal/client" "github.com/ryanfowler/fetch/internal/core" "github.com/ryanfowler/fetch/internal/curl" ) @@ -241,6 +244,14 @@ func isFlagVisibleOnOS(flagOS []string) bool { func Parse(args []string) (*App, error) { var app App + // Mark this before parsing so a one-shot stdin body can skip MIME sniffing + // even when --webtransport appears after -d @-. + for _, arg := range args { + if arg == "--webtransport" { + app.WebTransport = true + break + } + } cli := app.CLI() long, err := parseWithFlags(cli, args) @@ -273,6 +284,9 @@ func Parse(args []string) (*App, error) { if app.wsMessageModeSet && !app.WS { return &app, fmt.Errorf("'--ws-message-mode' requires a ws:// or wss:// URL") } + if err := ValidateWebTransport(&app); err != nil { + return &app, err + } if err := validateSchemeExclusives(&app, cli, long); err != nil { return &app, err @@ -290,6 +304,81 @@ func Parse(args []string) (*App, error) { return &app, nil } +// ValidateWebTransport validates mode-specific options after CLI and config +// values have been merged. It performs no I/O and is safe to call preflight. +func ValidateWebTransport(app *App) error { + if app == nil || !app.WebTransport { + if app != nil && (app.wtModeSet || app.wtDgramModeSet || len(app.WTProtocols) > 0) { + return errors.New("WebTransport options require --webtransport") + } + return nil + } + if app.WS { + return errors.New("WebSocket and WebTransport cannot be used together") + } + if app.InspectDNS || app.InspectTLS || app.Update || app.CheckUpdate || app.Skill || app.InstallSkill != "" || app.UninstallSkill != "" { + return errors.New("WebTransport cannot be combined with an inspection, update, or skill command") + } + if app.wtDgramModeSet && app.WTMode != core.WTDatagram { + return errors.New("--wt-datagram-mode requires --wt-mode datagram") + } + if name, ok := app.CLI().Options().Unsupported(ModeWebTransport); ok { + return fmt.Errorf("--%s cannot be used with WebTransport", name) + } + if app.URL == nil { + return nil + } + if !strings.EqualFold(app.URL.Scheme, "https") { + return errors.New("WebTransport requires an https:// URL") + } + if app.Cfg.HTTP == core.HTTP1 || app.Cfg.HTTP == core.HTTP2 { + return fmt.Errorf("WebTransport requires HTTP/3; cannot use %s", app.Cfg.HTTP.String()) + } + if app.Cfg.Format != core.FormatUnknown { + return errors.New("--format cannot be used with WebTransport") + } + if app.Cfg.TLSMax != nil && *app.Cfg.TLSMax < tls.VersionTLS13 { + return errors.New("WebTransport requires max-tls 1.3 or higher") + } + // These values can come from a merged config file, so registry IsSet + // checks alone are not sufficient here. + for _, unsupported := range []struct { + name string + set bool + }{ + {"compress", app.Cfg.Compress != core.CompressionUnknown}, + {"copy", app.Cfg.Copy != nil}, {"ignore-status", app.Cfg.IgnoreStatus != nil}, + {"no-encode", app.Cfg.NoEncode != nil}, {"redirects", app.Cfg.Redirects != nil}, + {"retry", app.Cfg.Retry != nil}, {"retry-delay", app.Cfg.RetryDelay != nil}, + {"retry-unsafe", app.Cfg.RetryUnsafe != nil}, + } { + if unsupported.set { + return fmt.Errorf("--%s cannot be used with WebTransport", unsupported.name) + } + } + if app.UnixSocket != "" { + return errors.New("WebTransport cannot be used with a unix socket") + } + if app.URL != nil { + decision, err := client.SelectProxy(app.Cfg.Proxy, app.URL) + if err != nil { + return err + } + if decision.URL != nil { + return errors.New("WebTransport cannot be used with a proxy") + } + } + for _, h := range app.Cfg.Headers { + if strings.EqualFold(h.Key, "Host") { + return errors.New("host header cannot be used with WebTransport") + } + if strings.EqualFold(h.Key, "WT-Available-Protocols") || strings.EqualFold(h.Key, "WT-Protocol") { + return fmt.Errorf("header %q cannot be supplied with WebTransport", h.Key) + } + } + return nil +} + func validateEquivalentAliases(app *App) error { if app.compressSet && app.noEncodeSet && app.explicitCompress != core.CompressionOff { return newExclusiveFlagsError("compress", "no-encode") diff --git a/internal/cli/registry.go b/internal/cli/registry.go index cde8966b..6f1fb2a0 100644 --- a/internal/cli/registry.go +++ b/internal/cli/registry.go @@ -31,6 +31,7 @@ const ( ModeGRPC OptionMode = "grpc" ModeGRPCDiscovery OptionMode = "grpc-discovery" ModeWebSocket OptionMode = "websocket" + ModeWebTransport OptionMode = "webtransport" ModeDNSInspection OptionMode = "dns-inspection" ModeTLSInspection OptionMode = "tls-inspection" ModeUpdate OptionMode = "update" @@ -288,6 +289,9 @@ func applyFlagDefinition(flag *Flag) { if schemes := websocketExcluded[flag.Long]; len(schemes) > 0 { flag.Schemes = append([]string(nil), schemes...) } + if webtransportExcluded[flag.Long] { + flag.UnsupportedIn = append(flag.UnsupportedIn, ModeWebTransport) + } if fromCurlOptions[flag.Long] { flag.FromCurl = true } @@ -295,7 +299,7 @@ func applyFlagDefinition(flag *Flag) { // existing flags useful to callers while specialized definitions below // narrow the modes where appropriate. if len(flag.Modes) == 0 { - flag.Modes = []OptionMode{ModeHTTP, ModeGRPC, ModeGRPCDiscovery, ModeWebSocket, ModeDNSInspection, ModeTLSInspection, ModeMetadata, ModeUpdate, ModeSkill} + flag.Modes = []OptionMode{ModeHTTP, ModeGRPC, ModeGRPCDiscovery, ModeWebSocket, ModeWebTransport, ModeDNSInspection, ModeTLSInspection, ModeMetadata, ModeUpdate, ModeSkill} } if flag.Default != "" { // The default is represented by the Flag.Default field; this branch is @@ -327,6 +331,9 @@ func applyFlagDefinition(flag *Flag) { flag.IgnoreLabel = def.IgnoreLabel flag.FromCurl = flag.FromCurl || def.FromCurl } + if webtransportExcluded[flag.Long] && !slices.Contains(flag.UnsupportedIn, ModeWebTransport) { + flag.UnsupportedIn = append(flag.UnsupportedIn, ModeWebTransport) + } } var websocketExcluded = map[string][]string{ @@ -339,6 +346,17 @@ var websocketExcluded = map[string][]string{ "digest": {"ws", "wss"}, "har": {"ws", "wss"}, } +var webtransportExcluded = map[string]bool{ + "article": true, "clobber": true, "compress": true, "copy": true, "discard": true, + "digest": true, "edit": true, "form": true, "grpc": true, "grpc-describe": true, + "grpc-list": true, "har": true, "ignore-status": true, "multipart": true, + "no-encode": true, "output": true, "proto-desc": true, "proto-file": true, + "proto-import": true, "range": true, "redirects": true, "remote-header-name": true, + "remote-name": true, "retry": true, "retry-delay": true, "retry-unsafe": true, + "unix": true, "xml": true, "format": true, "ws-message-mode": true, + "ws-interactive": true, +} + var fromCurlOptions = map[string]bool{ "method": true, "header": true, "data": true, "json": true, "xml": true, "form": true, "multipart": true, "basic": true, "bearer": true, "digest": true, @@ -375,11 +393,15 @@ var flagDefinitions = map[string]Flag{ "uninstall-skill": { Conflicts: []string{"skill", "install-skill"}, }, - "scope": {Requires: []string{"install-skill", "uninstall-skill"}}, - "force": {Requires: []string{"install-skill", "uninstall-skill"}}, - "ws-message-mode": {Modes: []OptionMode{ModeWebSocket}}, - "check-update": {Conflicts: []string{"update"}}, - "update": {Conflicts: []string{"check-update"}}, + "scope": {Requires: []string{"install-skill", "uninstall-skill"}}, + "force": {Requires: []string{"install-skill", "uninstall-skill"}}, + "ws-message-mode": {Modes: []OptionMode{ModeWebSocket}}, + "ws-interactive": {Modes: []OptionMode{ModeWebSocket}}, + "wt-mode": {Modes: []OptionMode{ModeWebTransport}}, + "wt-protocol": {Modes: []OptionMode{ModeWebTransport}, Repeatable: true}, + "wt-datagram-mode": {Modes: []OptionMode{ModeWebTransport}}, + "check-update": {Conflicts: []string{"update"}}, + "update": {Conflicts: []string{"check-update"}}, "aws-sigv4": {Conflicts: []string{"basic", "bearer", "digest"}, IgnoredIn: []OptionMode{ModeDNSInspection}, FromCurl: true}, "basic": {Conflicts: []string{"aws-sigv4", "bearer", "digest"}, IgnoredIn: []OptionMode{ModeDNSInspection}, FromCurl: true}, diff --git a/internal/cli/webtransport_test.go b/internal/cli/webtransport_test.go new file mode 100644 index 00000000..aed65181 --- /dev/null +++ b/internal/cli/webtransport_test.go @@ -0,0 +1,33 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/ryanfowler/fetch/internal/core" +) + +func TestWebTransportCLI(t *testing.T) { + app, err := Parse([]string{"--webtransport", "--wt-mode", "datagram", "--wt-protocol", "chat", "--wt-protocol", "chat-v2", "https://example.com/path"}) + if err != nil { + t.Fatal(err) + } + if !app.WebTransport || app.WTMode != core.WTDatagram || len(app.WTProtocols) != 2 { + t.Fatalf("app = %+v", app) + } + for _, args := range [][]string{ + {"--webtransport", "http://example.com"}, + {"--webtransport", "--http", "1", "https://example.com"}, + {"--webtransport", "--wt-datagram-mode", "binary", "https://example.com"}, + {"--webtransport", "--format", "off", "https://example.com"}, + {"--wt-mode", "stream", "https://example.com"}, + {"--webtransport", "--wt-protocol", "", "https://example.com"}, + } { + if _, err := Parse(args); err == nil { + t.Errorf("Parse(%v) succeeded", args) + } + } + if _, err := Parse([]string{"--webtransport", "--wt-protocol", "chat", "--wt-protocol", "chat", "https://example.com"}); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatalf("duplicate error = %v", err) + } +} diff --git a/internal/client/client.go b/internal/client/client.go index 3b36e560..3925fcb9 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -34,13 +34,19 @@ import ( // Client represents a wrapped HTTP client. type Client struct { - c *http.Client - maxRedirects int - initErr error - proxy *url.URL - httpVersion core.HTTPVersion - echMode core.ECHMode - resolver *resolver.Resolver + c *http.Client + maxRedirects int + initErr error + proxy *url.URL + httpVersion core.HTTPVersion + echMode core.ECHMode + resolver *resolver.Resolver + tlsConfig *tls.Config + connectTimeout time.Duration + webTransport bool + wtMu sync.Mutex + wtClosed bool + wtPackets []net.PacketConn } // RedirectHop represents a single redirect in the chain. @@ -193,6 +199,7 @@ type ClientConfig struct { TLSMin uint16 UnixSocket string ECH core.ECHMode + WebTransport bool } // NewClient returns an initialized Client given the provided configuration. @@ -230,121 +237,127 @@ func NewClient(cfg ClientConfig) *Client { } } - // Create the http.RoundTripper based on the configured HTTP version. + // Create the http.RoundTripper based on the configured HTTP version. A + // WebTransport client uses this Client only for request construction and + // cookie policy; its QUIC transport is created lazily by NewWebTransport. var transport http.RoundTripper - switch cfg.HTTP { - case core.HTTP2: - if cfg.H2C { - transport = getH2CTransport(baseDial, res, cfg.Proxy, cfg.ConnectTimeout) - } else { - transport = getHTTP2Transport(baseDial, res, cfg.Proxy, tlsConfig, cfg.ConnectTimeout, cfg.ECH) - } - case core.HTTP3: - transport = getHTTP3Transport(res, tlsConfig, cfg.ConnectTimeout, cfg.ECH) - default: - if useUnifiedProxyTransport(cfg.Proxy) { - transport = newUnifiedProxyTransport(cfg, baseDial, res, tlsConfig) - break - } - rt := &http.Transport{ - DisableCompression: true, - ForceAttemptHTTP2: cfg.HTTP != core.HTTP1, - Protocols: &http.Protocols{}, - TLSClientConfig: tlsConfig.Clone(), - } - if cfg.ECH != core.ECHUnknown && cfg.ECH != core.ECHOff && cfg.Proxy == nil { - // A custom TLS dial is required because net/http otherwise creates - // the tls.Config before the resolver can supply the origin's ECH - // configuration. Proxy-specific ECH wiring belongs to the proxy - // transport integration and must not weaken its certificate checks. - rt.DialTLSContext = newECHHTTPDialTLS(baseDial, res, tlsConfig, cfg.ECH, cfg.ConnectTimeout, cfg.HTTP) - } - - // net/http provides the HTTP proxy CONNECT machinery, but its SOCKS - // implementation treats socks5 and socks5h identically. It also uses - // the origin TLS configuration for an HTTPS proxy. Keep the transport - // as the single HTTP implementation, and replace only the first-hop - // dial for the schemes whose semantics need to be explicit. - transportProxy := func(req *http.Request) (*url.URL, error) { - selected, ok := selectedProxy(req.Context()) - if !ok { - var err error - selected, err = proxy(req) - if err != nil { - return nil, err - } + if cfg.WebTransport { + transport = &http.Transport{} + } else { + switch cfg.HTTP { + case core.HTTP2: + if cfg.H2C { + transport = getH2CTransport(baseDial, res, cfg.Proxy, cfg.ConnectTimeout) + } else { + transport = getHTTP2Transport(baseDial, res, cfg.Proxy, tlsConfig, cfg.ConnectTimeout, cfg.ECH) } - if selected == nil { - return nil, nil + case core.HTTP3: + transport = getHTTP3Transport(res, tlsConfig, cfg.ConnectTimeout, cfg.ECH) + default: + if useUnifiedProxyTransport(cfg.Proxy) { + transport = newUnifiedProxyTransport(cfg, baseDial, res, tlsConfig) + break } - switch strings.ToLower(selected.Scheme) { - case "https": - return httpsProxyAsHTTP(selected), nil - case "socks5", "socks5h": - // SOCKS destinations are carried by DialContext rather than - // net/http's SOCKS implementation so socks5 can resolve - // locally and socks5h can preserve the hostname. - return nil, nil - default: - return selected, nil + rt := &http.Transport{ + DisableCompression: true, + ForceAttemptHTTP2: cfg.HTTP != core.HTTP1, + Protocols: &http.Protocols{}, + TLSClientConfig: tlsConfig.Clone(), } - } - dial := wrapDialWithConnectTimeout(baseDial, cfg.ConnectTimeout) - if cfg.ConnectTimeout <= 0 { - dial = baseDial - } - if cfg.Proxy != nil { - switch strings.ToLower(cfg.Proxy.Scheme) { - case "socks5", "socks5h": - transportProxy = func(*http.Request) (*url.URL, error) { return nil, nil } - dial = newSOCKS5Dialer(baseDial, res, cfg.Proxy, strings.EqualFold(cfg.Proxy.Scheme, "socks5"), cfg.ConnectTimeout) - case "https": - transportProxy = func(*http.Request) (*url.URL, error) { - return httpsProxyAsHTTP(cfg.Proxy), nil + if cfg.ECH != core.ECHUnknown && cfg.ECH != core.ECHOff && cfg.Proxy == nil { + // A custom TLS dial is required because net/http otherwise creates + // the tls.Config before the resolver can supply the origin's ECH + // configuration. Proxy-specific ECH wiring belongs to the proxy + // transport integration and must not weaken its certificate checks. + rt.DialTLSContext = newECHHTTPDialTLS(baseDial, res, tlsConfig, cfg.ECH, cfg.ConnectTimeout, cfg.HTTP) + } + + // net/http provides the HTTP proxy CONNECT machinery, but its SOCKS + // implementation treats socks5 and socks5h identically. It also uses + // the origin TLS configuration for an HTTPS proxy. Keep the transport + // as the single HTTP implementation, and replace only the first-hop + // dial for the schemes whose semantics need to be explicit. + transportProxy := func(req *http.Request) (*url.URL, error) { + selected, ok := selectedProxy(req.Context()) + if !ok { + var err error + selected, err = proxy(req) + if err != nil { + return nil, err + } + } + if selected == nil { + return nil, nil + } + switch strings.ToLower(selected.Scheme) { + case "https": + return httpsProxyAsHTTP(selected), nil + case "socks5", "socks5h": + // SOCKS destinations are carried by DialContext rather than + // net/http's SOCKS implementation so socks5 can resolve + // locally and socks5h can preserve the hostname. + return nil, nil + default: + return selected, nil } - dial = newHTTPSProxyDialer(baseDial, cfg.Proxy, cfg.ConnectTimeout) - default: - transportProxy = proxy } - } - if cfg.Proxy == nil { - // The selected environment proxy is carried in the request context. - // http.Transport passes that context to DialContext, so concurrent - // requests cannot change one another's first hop. - wrappedDial := dial - dial = func(ctx context.Context, network, address string) (net.Conn, error) { - selected, ok := selectedProxy(ctx) - if ok && selected != nil { - switch strings.ToLower(selected.Scheme) { - case "socks5", "socks5h": - return newSOCKS5Dialer(baseDial, res, selected, strings.EqualFold(selected.Scheme, "socks5"), cfg.ConnectTimeout)(ctx, network, address) - case "https": - return newHTTPSProxyDialer(baseDial, selected, cfg.ConnectTimeout)(ctx, network, address) + dial := wrapDialWithConnectTimeout(baseDial, cfg.ConnectTimeout) + if cfg.ConnectTimeout <= 0 { + dial = baseDial + } + if cfg.Proxy != nil { + switch strings.ToLower(cfg.Proxy.Scheme) { + case "socks5", "socks5h": + transportProxy = func(*http.Request) (*url.URL, error) { return nil, nil } + dial = newSOCKS5Dialer(baseDial, res, cfg.Proxy, strings.EqualFold(cfg.Proxy.Scheme, "socks5"), cfg.ConnectTimeout) + case "https": + transportProxy = func(*http.Request) (*url.URL, error) { + return httpsProxyAsHTTP(cfg.Proxy), nil } + dial = newHTTPSProxyDialer(baseDial, cfg.Proxy, cfg.ConnectTimeout) + default: + transportProxy = proxy } - return wrappedDial(ctx, network, address) } - } - rt.Proxy = transportProxy - rt.DialContext = dial - rt.Protocols.SetHTTP1(true) - rt.Protocols.SetHTTP2(cfg.HTTP != core.HTTP1) - transport = rt - if cfg.Proxy == nil { - transport = &proxyTransport{base: rt, selectProxy: proxy} - } - // Automatic HTTP/3 is only safe for direct HTTPS requests. The - // wrapper delegates HTTP, proxy, and Unix-socket requests to this - // ordinary transport, and prepares exactly one complete TCP/TLS or - // QUIC connection before sending an eligible request. - if cfg.HTTP == core.HTTPDefault && cfg.Proxy == nil && cfg.UnixSocket == "" { - // Environment-selected proxies are not eligible for automatic H3. - // Keep the ordinary transport visible in that case so proxy setup - // remains the single source of truth. - autoHTTPSProxy, httpsProxyErr := ProxyForURL(nil, &url.URL{Scheme: "https", Host: "example.com"}) - autoHTTPProxy, httpProxyErr := ProxyForURL(nil, &url.URL{Scheme: "http", Host: "example.com"}) - if httpsProxyErr == nil && httpProxyErr == nil && autoHTTPSProxy == nil && autoHTTPProxy == nil { - transport = newAutomaticHTTP3Transport(rt, res, cfg.ConnectTimeout, tlsConfig, cfg.ECH) + if cfg.Proxy == nil { + // The selected environment proxy is carried in the request context. + // http.Transport passes that context to DialContext, so concurrent + // requests cannot change one another's first hop. + wrappedDial := dial + dial = func(ctx context.Context, network, address string) (net.Conn, error) { + selected, ok := selectedProxy(ctx) + if ok && selected != nil { + switch strings.ToLower(selected.Scheme) { + case "socks5", "socks5h": + return newSOCKS5Dialer(baseDial, res, selected, strings.EqualFold(selected.Scheme, "socks5"), cfg.ConnectTimeout)(ctx, network, address) + case "https": + return newHTTPSProxyDialer(baseDial, selected, cfg.ConnectTimeout)(ctx, network, address) + } + } + return wrappedDial(ctx, network, address) + } + } + rt.Proxy = transportProxy + rt.DialContext = dial + rt.Protocols.SetHTTP1(true) + rt.Protocols.SetHTTP2(cfg.HTTP != core.HTTP1) + transport = rt + if cfg.Proxy == nil { + transport = &proxyTransport{base: rt, selectProxy: proxy} + } + // Automatic HTTP/3 is only safe for direct HTTPS requests. The + // wrapper delegates HTTP, proxy, and Unix-socket requests to this + // ordinary transport, and prepares exactly one complete TCP/TLS or + // QUIC connection before sending an eligible request. + if cfg.HTTP == core.HTTPDefault && cfg.Proxy == nil && cfg.UnixSocket == "" { + // Environment-selected proxies are not eligible for automatic H3. + // Keep the ordinary transport visible in that case so proxy setup + // remains the single source of truth. + autoHTTPSProxy, httpsProxyErr := ProxyForURL(nil, &url.URL{Scheme: "https", Host: "example.com"}) + autoHTTPProxy, httpProxyErr := ProxyForURL(nil, &url.URL{Scheme: "http", Host: "example.com"}) + if httpsProxyErr == nil && httpProxyErr == nil && autoHTTPSProxy == nil && autoHTTPProxy == nil { + transport = newAutomaticHTTP3Transport(rt, res, cfg.ConnectTimeout, tlsConfig, cfg.ECH) + } } } } @@ -466,7 +479,7 @@ func NewClient(cfg ClientConfig) *Client { initErr = setErr } } - if cfg.HTTP == core.HTTP3 && cfg.Proxy != nil { + if (cfg.HTTP == core.HTTP3 || cfg.WebTransport) && cfg.Proxy != nil { initErr = errors.New("HTTP/3 cannot be used with a proxy") } if initErr == nil { @@ -476,19 +489,26 @@ func NewClient(cfg ClientConfig) *Client { initErr = errors.New("ECH cannot be used with cleartext HTTP/2") } if initErr == nil { - initErr = core.ValidateECHPolicy(cfg.ECH, cfg.HTTP, cfg.TLSMin, cfg.TLSMax) + version := cfg.HTTP + if cfg.WebTransport { + version = core.HTTPDefault + } + initErr = core.ValidateECHPolicy(cfg.ECH, version, cfg.TLSMin, cfg.TLSMax) } if initErr == nil && cfg.HTTP == core.HTTP3 && cfg.TLSMax != 0 && cfg.TLSMax < tls.VersionTLS13 { initErr = errors.New("HTTP/3 requires max-tls 1.3 or higher") } return &Client{ - c: client, - maxRedirects: maxRedirects, - initErr: initErr, - proxy: cfg.Proxy, - httpVersion: cfg.HTTP, - echMode: cfg.ECH, - resolver: res, + c: client, + maxRedirects: maxRedirects, + initErr: initErr, + proxy: cfg.Proxy, + httpVersion: cfg.HTTP, + echMode: cfg.ECH, + resolver: res, + tlsConfig: tlsConfig.Clone(), + connectTimeout: cfg.ConnectTimeout, + webTransport: cfg.WebTransport, } } @@ -868,6 +888,14 @@ func (t *http3TimingTransport) Close() error { // Close closes the underlying transport, releasing any resources. func (c *Client) Close() error { var closeErr error + c.wtMu.Lock() + c.wtClosed = true + packets := c.wtPackets + c.wtPackets = nil + c.wtMu.Unlock() + for _, packet := range packets { + closeErr = errors.Join(closeErr, packet.Close()) + } if idleCloser, ok := c.c.Transport.(interface{ CloseIdleConnections() }); ok { idleCloser.CloseIdleConnections() } @@ -929,6 +957,18 @@ func (c *Client) SetJar(jar http.CookieJar) { // before sending a request. It is intended for dry-run metadata, which is // rendered before Client.Do would normally apply the jar. The caller must not // call it more than once for the same request. +// SetResponseCookies applies cookies returned by a direct protocol handshake. +// net/http normally performs this step inside Client.Do. +func (c *Client) SetResponseCookies(reqURL *url.URL, resp *http.Response) { + if c == nil || c.c == nil || c.c.Jar == nil || reqURL == nil || resp == nil { + return + } + cookies := resp.Cookies() + if len(cookies) > 0 { + c.c.Jar.SetCookies(reqURL, cookies) + } +} + func (c *Client) ApplyJarCookies(req *http.Request) *http.Request { if c.c.Jar == nil { return req diff --git a/internal/client/webtransport.go b/internal/client/webtransport.go new file mode 100644 index 00000000..49e2257f --- /dev/null +++ b/internal/client/webtransport.go @@ -0,0 +1,227 @@ +package client + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http/httptrace" + "strconv" + "strings" + + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" + "github.com/quic-go/webtransport-go" + "github.com/ryanfowler/fetch/internal/core" + "github.com/ryanfowler/fetch/internal/resolver" +) + +// NewWebTransport returns the configured WebTransport transport. The caller +// owns the returned transport through Client.Close; packet sockets are kept +// private so a failed or canceled session cannot leak them. +func (c *Client) NewWebTransport(protocols []string) (*webtransport.Transport, error) { + if c == nil { + return nil, errors.New("nil client") + } + if !c.webTransport { + return nil, errors.New("client was not configured for WebTransport") + } + if c.initErr != nil { + return nil, c.initErr + } + t := &webtransport.Transport{ + Config: &webtransport.Config{MaxIncomingStreams: -1, MaxIncomingUniStreams: -1}, + TLSClientConfig: func() *tls.Config { + cfg := c.tlsConfig.Clone() + cfg.NextProtos = []string{http3.NextProtoH3} + return cfg + }(), + QUICConfig: &quic.Config{ + EnableDatagrams: true, + EnableStreamResetPartialDelivery: true, + }, + ApplicationProtocols: append([]string(nil), protocols...), + } + t.DialAddr = c.webTransportDial + return t, nil +} + +func (c *Client) webTransportDial(ctx context.Context, addr string, tlsCfg *tls.Config, qcfg *quic.Config) (*quic.Conn, error) { + connectCtx, cancel := connectContext(ctx, c.connectTimeout, "DNS/QUIC/TLS connect") + defer cancel() + host, port, err := net.SplitHostPort(addr) + if err != nil { + host, port = strings.Trim(host, "[]"), "443" + host = strings.ReplaceAll(host, "%25", "%") + } + trace := httptrace.ContextClientTrace(connectCtx) + if trace != nil && trace.DNSStart != nil { + trace.DNSStart(httptrace.DNSStartInfo{Host: host}) + } + _, hasResolve, err := c.resolver.ResolveAddressOverride("udp", host, port) + if err != nil { + return nil, err + } + endpoint, err := c.resolver.ResolveAddress(connectCtx, "udp", net.JoinHostPort(host, port)) + if trace != nil && trace.DNSDone != nil { + info := httptrace.DNSDoneInfo{Err: err} + if err == nil { + info.Addrs = append([]net.IPAddr(nil), endpoint.Addrs...) + } + trace.DNSDone(info) + } + if err != nil { + return nil, err + } + if !hasResolve { + discoveryPort := 443 + if parsed, parseErr := strconv.Atoi(port); parseErr == nil && parsed > 0 && parsed <= 65535 { + discoveryPort = parsed + } + discovery, discoveryErr := c.resolver.DiscoverHTTPS(connectCtx, host, uint16(discoveryPort), nil) + if discoveryErr != nil && (c.echMode == core.ECHOn || resolver.IsAuthenticatedDiscoveryFailure(discoveryErr)) { + return nil, discoveryErr + } + if discoveryErr == nil { + for i := range discovery.Candidates { + candidate := &discovery.Candidates[i] + supportsH3 := false + for _, alpn := range candidate.ALPN { + if string(alpn) == "h3" { + supportsH3 = true + break + } + } + if supportsH3 { // selected HTTP/3 service + if len(candidate.Addresses) > 0 { + endpoint.Addrs = candidate.Addresses + } + if candidate.Port != 0 { + endpoint.Port = strconv.Itoa(int(candidate.Port)) + } + break + } + } + } + } + portNumber, err := strconv.Atoi(endpoint.Port) + if err != nil || portNumber < 1 || portNumber > 65535 { + return nil, fmt.Errorf("invalid WebTransport port %q", endpoint.Port) + } + baseTLS := tlsCfg.Clone() + if baseTLS.ServerName == "" { + baseTLS.ServerName = core.TLSVerificationName(host) + } + // ECH discovery is deliberately performed on the same resolver and within + // the same connection budget as address resolution. + if c.echMode != core.ECHUnknown && c.echMode != core.ECHOff { + echResolver := c.resolver + if hasResolve { + if c.echMode == core.ECHOn { + return nil, ErrECHConfigUnavailable + } + echResolver = nil // authoritative --resolve does not trigger discovery + } + ech, echErr := DiscoverECHForConnection(connectCtx, echResolver, host, endpoint.Port, baseTLS, c.echMode, core.HTTP3) + if echErr != nil { + return nil, echErr + } + baseTLS = ech.TLSConfig() + if targetHost, targetPort := ech.Target(); targetHost != "" { + host, endpoint.Port = targetHost, targetPort + portNumber, _ = strconv.Atoi(targetPort) + } + if addresses := ech.Addresses(); len(addresses) > 0 { + endpoint.Addrs = addresses + } + } + if len(endpoint.Addrs) == 0 { + return nil, fmt.Errorf("no addresses found for %s", host) + } + winner, err := raceQUIC(connectCtx, endpoint.Addrs, portNumber, baseTLS, qcfg, trace, func(p net.PacketConn) { + c.wtMu.Lock() + if c.wtClosed { + c.wtMu.Unlock() + _ = p.Close() + return + } + c.wtPackets = append(c.wtPackets, p) + c.wtMu.Unlock() + }) + if err != nil { + return nil, err + } + return winner, nil +} + +// raceQUIC keeps the QUIC attempt and its packet socket together. A socket is +// caller-owned even after quic.Conn.Close, so both are closed on every loss. +func raceQUIC(ctx context.Context, addresses []net.IPAddr, port int, tlsCfg *tls.Config, qcfg *quic.Config, trace *httptrace.ClientTrace, own func(net.PacketConn)) (*quic.Conn, error) { + type attemptResult struct { + conn *quic.Conn + packet net.PacketConn + } + result, err := resolver.RaceCandidates(ctx, addresses, func(attemptCtx context.Context, ip net.IPAddr) (attemptResult, error) { + address := core.JoinIPHostPort(ip, strconv.Itoa(port)) + if trace != nil && trace.ConnectStart != nil { + trace.ConnectStart("udp", address) + } + packet, err := (&net.ListenConfig{}).ListenPacket(attemptCtx, "udp", ":0") + if err != nil { + return attemptResult{}, err + } + cfg := qcfg + if cfg != nil { + cfg = cfg.Clone() + } else { + cfg = &quic.Config{} + } + cfg.EnableDatagrams = true + cfg.EnableStreamResetPartialDelivery = true + if trace != nil && trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } + conn, err := quic.DialEarly(attemptCtx, packet, &net.UDPAddr{IP: ip.IP, Port: port, Zone: ip.Zone}, tlsCfg.Clone(), cfg) + if err == nil { + select { + case <-conn.HandshakeComplete(): + case <-attemptCtx.Done(): + err = context.Cause(attemptCtx) + } + } + if trace != nil && trace.TLSHandshakeDone != nil { + var state tls.ConnectionState + if conn != nil && err == nil { + state = conn.ConnectionState().TLS + } + trace.TLSHandshakeDone(state, err) + } + if trace != nil && trace.ConnectDone != nil { + trace.ConnectDone("udp", address, err) + } + if err != nil { + if conn != nil { + _ = conn.CloseWithError(0, "handshake failed") + } + _ = packet.Close() + return attemptResult{}, err + } + return attemptResult{conn: conn, packet: packet}, nil + }, func(loser attemptResult) { + if loser.conn != nil { + _ = loser.conn.CloseWithError(0, "address race lost") + } + if loser.packet != nil { + _ = loser.packet.Close() + } + }) + if err != nil { + return nil, err + } + own(result.packet) + if trace != nil && trace.GotConn != nil { + trace.GotConn(httptrace.GotConnInfo{Conn: traceAddrConn{remote: result.conn.RemoteAddr()}}) + } + return result.conn, nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 0b9f7ae6..6d8d4270 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -379,7 +379,13 @@ func (c *Config) Merge(c2 *Config) []string { // Validate checks cross-option constraints that can only be evaluated after // CLI, global, and host-specific configuration have been merged. -func (c *Config) Validate() error { +func (c *Config) Validate() error { return c.validate(false) } + +// ValidateForWebTransport applies the same config checks while allowing +// mandatory ECH on WebTransport's explicit HTTP/3 transport. +func (c *Config) ValidateForWebTransport() error { return c.validate(true) } + +func (c *Config) validate(webTransport bool) error { var tlsMin, tlsMax uint16 if c.TLSMin != nil { tlsMin = *c.TLSMin @@ -393,10 +399,14 @@ func (c *Config) Validate() error { if err := core.ValidateTLSVersions(tlsMin, tlsMax); err != nil { return err } - if c.HTTP == core.HTTP3 && c.TLSMax != nil && *c.TLSMax < tls.VersionTLS13 { + if (c.HTTP == core.HTTP3 || webTransport) && c.TLSMax != nil && *c.TLSMax < tls.VersionTLS13 { return fmt.Errorf("HTTP/3 requires max-tls 1.3 or higher") } - if err := core.ValidateECHPolicy(c.ECH, c.HTTP, tlsMin, tlsMax); err != nil { + echHTTP := c.HTTP + if webTransport { + echHTTP = core.HTTPDefault + } + if err := core.ValidateECHPolicy(c.ECH, echHTTP, tlsMin, tlsMax); err != nil { return err } if c.KeyData != nil && c.CertData == nil { diff --git a/internal/core/core.go b/internal/core/core.go index 44907f8e..fbff756f 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -99,6 +99,36 @@ const ( ImageOff ) +// WTMode selects the WebTransport data path. +type WTMode int + +const ( + WTStream WTMode = iota + WTDatagram +) + +func (m WTMode) String() string { + if m == WTDatagram { + return "datagram" + } + return "stream" +} + +// WTDatagramMode selects how stdin is split into datagrams. +type WTDatagramMode int + +const ( + WTDatagramLines WTDatagramMode = iota + WTDatagramBinary +) + +func (m WTDatagramMode) String() string { + if m == WTDatagramBinary { + return "binary" + } + return "lines" +} + // WSMessageMode controls how WebSocket payloads are interpreted. type WSMessageMode int diff --git a/internal/core/limits.go b/internal/core/limits.go index 54be3369..7427b701 100644 --- a/internal/core/limits.go +++ b/internal/core/limits.go @@ -18,6 +18,8 @@ const ( MaxWebSocketMessageBytes int64 = 16 << 20 MaxWebSocketPipedTextLine int64 = 16 << 20 MaxWebSocketInteractiveEntry int64 = 16 << 20 + MaxWebTransportDatagramLine int64 = 64 << 10 + MaxWebTransportBinaryChunk = 1 << 10 MaxStreamingRecordBytes int64 = 16 << 20 MaxCompositeMaterialization int64 = 16 << 20 MaxGRPCMessageBytes int64 = 64 << 20 diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index 11456a66..c673c3d9 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -85,6 +85,7 @@ type Request struct { NoPager bool Pager core.PagerMode Method string + MethodExplicit bool Multipart *multipart.Multipart Output string PrinterHandle *core.Handle @@ -111,6 +112,10 @@ type Request struct { URL *url.URL Verbosity core.Verbosity WS bool + WebTransport bool + WTMode core.WTMode + WTDgramMode core.WTDatagramMode + WTProtocols []string WSInteractive core.WSInteractiveMode WSMessageMode core.WSMessageMode SchemelessURL bool @@ -293,7 +298,14 @@ func fetch(ctx context.Context, r *Request) (int, error) { } }() - // 4. WebSocket: branch to handleWebSocket before edit/gRPC/retry. The + // 4. WebTransport and WebSocket are persistent protocol branches. Their + // request body is application data and must remain deferred until after + // their handshakes. + if r.WebTransport { + return handleWebTransport(ctx, r, c, req) + } + + // 5. WebSocket: branch to handleWebSocket before edit/gRPC/retry. The // session save is deferred above, so handshake cookie changes persist even // when the message loop or a later validation step fails. if r.WS { diff --git a/internal/fetch/grpc_reflection.go b/internal/fetch/grpc_reflection.go index 9b6a3966..29695ae8 100644 --- a/internal/fetch/grpc_reflection.go +++ b/internal/fetch/grpc_reflection.go @@ -830,6 +830,9 @@ func newClient(r *Request) *client.Client { if r.WS { httpVersion = core.HTTP1 } + if r.WebTransport { + httpVersion = core.HTTP3 + } return client.NewClient(client.ClientConfig{ CACerts: r.CACerts, ClientCert: r.ClientCert, @@ -846,6 +849,7 @@ func newClient(r *Request) *client.Client { TLSMax: r.TLSMax, TLSMin: r.TLSMin, UnixSocket: r.UnixSocket, + WebTransport: r.WebTransport, }) } diff --git a/internal/fetch/webtransport.go b/internal/fetch/webtransport.go new file mode 100644 index 00000000..ad6b5dbe --- /dev/null +++ b/internal/fetch/webtransport.go @@ -0,0 +1,241 @@ +package fetch + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptrace" + "os" + "strings" + + "github.com/dunglas/httpsfv" + "github.com/quic-go/webtransport-go" + "github.com/ryanfowler/fetch/internal/body" + "github.com/ryanfowler/fetch/internal/client" + "github.com/ryanfowler/fetch/internal/core" + "github.com/ryanfowler/fetch/internal/wt" +) + +func handleWebTransport(ctx context.Context, r *Request, c *client.Client, req *http.Request) (int, error) { + if r.MethodExplicit && req.Method != http.MethodConnect { + p := r.PrinterHandle.Stderr() + core.WriteWarningMsgIf(p, "WebTransport requires CONNECT; ignoring method "+req.Method, r.Verbosity == core.VSilent) + req.Method = http.MethodConnect + } + p := r.PrinterHandle.Stderr() + if r.Timing { + core.WriteWarningMsgIf(p, "--timing is not supported for WebTransport connections", r.Verbosity == core.VSilent) + } + if r.Pager != core.PagerUnknown || r.NoPager { + core.WriteWarningMsgIf(p, "pager does not apply to WebTransport output", r.Verbosity == core.VSilent) + } + if r.Image != core.ImageUnknown { + core.WriteWarningMsgIf(p, "image rendering does not apply to WebTransport output", r.Verbosity == core.VSilent) + } + + protocols := append([]string(nil), r.WTProtocols...) + if len(protocols) > 0 { + value, err := marshalWTProtocols(protocols) + if err != nil { + return 1, err + } + req.Header.Set("WT-Available-Protocols", value) + } + // A CONNECT body is never sent. Preserve it as deferred application input. + var initialReader io.Reader + var initialSet bool + var preview *body.Body + initialOwned := false + if source, ok := body.SourceFromContext(req.Context()); ok { + initialSet = true + if r.DryRun { + preview = source + } + } + if req.Body != nil && req.Body != http.NoBody { + if !r.DryRun { + initialReader = req.Body + initialOwned = true + } + req.Body = http.NoBody + req.GetBody = nil + req.ContentLength = 0 + } + defer func() { + if initialOwned { + if closer, ok := initialReader.(io.Closer); ok { + _ = closer.Close() + } + } + }() + // Transport.Dial is not an http.Client request, so apply the active jar + // explicitly. Do this once, before signing, just like net/http. + req = c.ApplyJarCookies(req) + if err := signAWSRequest(r, req); err != nil { + return 1, err + } + // This check is network-free and also catches environment and system + // proxies. It must run before dry-run output and before the QUIC dial. + if err := c.ValidateTransport(req); err != nil { + return 1, err + } + + if r.Verbosity >= core.VExtraVerbose || r.DryRun { + p := r.PrinterHandle.Stderr() + printRequestMetadataWithURL(p, req, core.HTTP3, r.Verbosity, r.DryRun) + if r.Verbosity >= core.VDebug { + printProxyMetadata(p, r.Proxy, req.URL) + printResolverMetadata(p, c) + } + p.WriteString("webtransport mode: ") + p.WriteString(r.WTMode.String()) + p.WriteString("\n") + if len(protocols) > 0 { + p.WriteString("webtransport protocols: ") + p.WriteString(core.TerminalSafeText(strings.Join(protocols, ", "))) + p.WriteString("\n") + } + p.Flush() + if r.DryRun { + if preview != nil { + if err := printDryRunBodyPreview(p, preview, r.Verbosity == core.VSilent); err != nil { + _ = preview.Close() + return 1, err + } + _ = preview.Close() + } + return 0, nil + } + } + dialCtx := ctx + if r.Timeout > 0 { + var cancel context.CancelFunc + dialCtx, cancel = context.WithTimeout(ctx, r.Timeout) + defer cancel() + } + if r.Verbosity >= core.VDebug { + trace, _ := newDebugTrace(r.PrinterHandle.Stderr()) + dialCtx = httptrace.WithClientTrace(dialCtx, trace) + } + t, err := c.NewWebTransport(protocols) + if err != nil { + return 1, err + } + resp, session, err := t.Dial(dialCtx, req.URL.String(), req.Header) + if resp != nil { + c.SetResponseCookies(req.URL, resp) + } + if r.Verbosity >= core.VNormal && resp != nil { + p := r.PrinterHandle.Stderr() + printResponseMetadata(p, r.Verbosity, resp) + if r.Verbosity >= core.VVerbose { + if selected := resp.Header.Get("WT-Protocol"); selected != "" { + p.WriteString("webtransport protocol: ") + p.WriteString(core.TerminalSafeText(selected)) + p.WriteString("\n") + } + } + p.Flush() + } + if err != nil { + return 1, webtransportHandshakeError(resp, err) + } + if session == nil { + return 1, errors.New("WebTransport handshake succeeded without a session") + } + defer session.CloseWithError(0, "") + state := session.SessionState() + if r.Verbosity >= core.VVerbose && state.ApplicationProtocol != "" { + p := r.PrinterHandle.Stderr() + p.WriteString("webtransport protocol: ") + p.WriteString(core.TerminalSafeText(state.ApplicationProtocol)) + p.WriteString("\n") + p.Flush() + } + + stdin := io.Reader(nil) + if initialReader == nil || isReplayableInitial(req) { + if info, statErr := os.Stdin.Stat(); statErr == nil && (info.Size() > 0 || info.Mode()&os.ModeNamedPipe != 0) { + stdin = os.Stdin + } + } + initialOwned = false + stdout := &flushPrinterWriter{printer: r.PrinterHandle.Stdout()} + return wtStatus(wt.Run(ctx, wt.Config{Session: webTransportSession{session}, Stdin: stdin, Stdout: stdout, Mode: r.WTMode, DatagramMode: r.WTDgramMode, InitialPayloadSet: initialSet && initialReader == nil, InitialPayload: nil, InitialReader: initialReader, TerminalOutput: core.IsStdoutTerm && r.WTMode == core.WTStream})) +} + +type flushPrinterWriter struct{ printer *core.Printer } + +func (w *flushPrinterWriter) Write(p []byte) (int, error) { + n, err := w.printer.Write(p) + if err != nil { + return n, err + } + if err = w.printer.Flush(); err != nil { + return n, err + } + return n, nil +} + +func isReplayableInitial(req *http.Request) bool { + source, ok := body.SourceFromContext(req.Context()) + return ok && source.Replayable() +} + +func wtStatus(err error) (int, error) { + if err != nil { + return 1, err + } + return 0, nil +} + +func marshalWTProtocols(protocols []string) (string, error) { + list := make(httpsfv.List, 0, len(protocols)) + for _, protocol := range protocols { + list = append(list, httpsfv.NewItem(protocol)) + } + value, err := httpsfv.Marshal(list) + if err != nil { + return "", fmt.Errorf("invalid WebTransport application protocol: %w", err) + } + return value, nil +} + +type webTransportSession struct{ session *webtransport.Session } + +func (s webTransportSession) OpenStream(ctx context.Context) (io.ReadWriteCloser, error) { + return s.session.OpenStreamSync(ctx) +} +func (s webTransportSession) SendDatagram(data []byte) error { return s.session.SendDatagram(data) } +func (s webTransportSession) ReceiveDatagram(ctx context.Context) ([]byte, error) { + return s.session.ReceiveDatagram(ctx) +} +func (s webTransportSession) Close() error { return s.session.CloseWithError(0, "") } + +func webtransportHandshakeError(resp *http.Response, err error) error { + message := "WebTransport handshake failed" + if resp != nil { + message += " with " + core.TerminalSafeText(resp.Status) + } + if err != nil { + message += ": " + core.RedactedErrorText(err) + } + if resp == nil || resp.Body == nil { + return errors.New(message) + } + const limit = 1024 + data, readErr := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + _ = resp.Body.Close() + if len(data) > limit { + data = data[:limit] + } + if len(data) > 0 { + message += fmt.Sprintf(": response excerpt %q", core.TerminalSafeText(string(data))) + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + message += ": " + core.TerminalSafeText(readErr.Error()) + } + return errors.New(message) +} diff --git a/internal/wt/wt.go b/internal/wt/wt.go new file mode 100644 index 00000000..0c3ab80b --- /dev/null +++ b/internal/wt/wt.go @@ -0,0 +1,396 @@ +// Package wt contains the protocol-independent WebTransport session loop. +package wt + +import ( + "bufio" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "unicode/utf8" + + "github.com/quic-go/webtransport-go" + "github.com/ryanfowler/fetch/internal/core" +) + +// Session is the small part of a WebTransport session needed by the CLI. +type Session interface { + OpenStream(context.Context) (io.ReadWriteCloser, error) + SendDatagram([]byte) error + ReceiveDatagram(context.Context) ([]byte, error) + Close() error +} + +type Config struct { + Session Session + Stdin io.Reader + Stdout io.Writer + Mode core.WTMode + DatagramMode core.WTDatagramMode + InitialPayload []byte + InitialPayloadSet bool + InitialReader io.Reader + TerminalOutput bool +} + +func Run(ctx context.Context, cfg Config) error { + if cfg.Session == nil || cfg.Stdout == nil { + return errors.New("WebTransport session and stdout are required") + } + if cfg.Mode == core.WTDatagram { + return runDatagrams(ctx, cfg) + } + return runStream(ctx, cfg) +} + +type streamResult struct{ err error } + +func runStream(ctx context.Context, cfg Config) error { + stream, err := cfg.Session.OpenStream(ctx) + if err != nil { + return fmt.Errorf("open WebTransport stream: %w", err) + } + defer stream.Close() + workCtx, cancel := context.WithCancel(ctx) + defer cancel() + stopWatch := make(chan struct{}) + watchDone := make(chan struct{}) + go func() { + defer close(watchDone) + select { + case <-workCtx.Done(): + _ = cfg.Session.Close() + closeInput(cfg.InitialReader) + closeInput(cfg.Stdin) + case <-stopWatch: + } + }() + defer func() { close(stopWatch); <-watchDone }() + writeDone := make(chan streamResult, 1) + go func() { + err := writeStream(workCtx, stream, cfg) + if err != nil { + _ = cfg.Session.Close() + cancel() + } + writeDone <- streamResult{err} + }() + + reader := io.Reader(stream) + var output io.Writer = cfg.Stdout + var safe *terminalWriter + if cfg.TerminalOutput { + safe = &terminalWriter{dst: cfg.Stdout} + output = safe + } + buf := make([]byte, 32*1024) + readErr := error(nil) + for { + n, read := reader.Read(buf) + if n > 0 { + if _, write := output.Write(buf[:n]); write != nil { + readErr = write + cancel() + _ = cfg.Session.Close() + break + } + } + if read == io.EOF { + break + } + if read != nil { + readErr = read + cancel() + break + } + } + if safe != nil && readErr == nil { + readErr = safe.Flush() + } + write := (<-writeDone).err + if readErr != nil && !isCleanClose(readErr) { + _ = cfg.Session.Close() + return fmt.Errorf("read WebTransport stream: %w", readErr) + } + if write != nil { + _ = cfg.Session.Close() + return write + } + return nil +} + +func writeStream(ctx context.Context, stream io.WriteCloser, cfg Config) error { + write := func(r io.Reader) error { + if r == nil { + return nil + } + _, err := io.CopyBuffer(stream, r, make([]byte, 32*1024)) + if err != nil { + return fmt.Errorf("write WebTransport stream: %w", err) + } + return nil + } + if cfg.InitialPayloadSet { + if _, err := stream.Write(cfg.InitialPayload); err != nil { + return fmt.Errorf("write initial WebTransport payload: %w", err) + } + } + if cfg.InitialReader != nil { + if err := write(cfg.InitialReader); err != nil { + closeInput(cfg.InitialReader) + return err + } + closeInput(cfg.InitialReader) + } + if cfg.Stdin != nil { + if err := writeContext(ctx, stream, cfg.Stdin); err != nil { + closeInput(cfg.Stdin) + return err + } + closeInput(cfg.Stdin) + } + if err := stream.Close(); err != nil { + return fmt.Errorf("close WebTransport stream: %w", err) + } + return nil +} + +func writeContext(ctx context.Context, dst io.Writer, src io.Reader) error { + buf := make([]byte, 32*1024) + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + n, err := src.Read(buf) + if n > 0 { + if _, e := dst.Write(buf[:n]); e != nil { + return e + } + } + if err == io.EOF { + return nil + } + if err != nil { + return err + } + } +} + +func runDatagrams(ctx context.Context, cfg Config) error { + workCtx, cancel := context.WithCancel(ctx) + defer cancel() + stopWatch := make(chan struct{}) + watchDone := make(chan struct{}) + go func() { + defer close(watchDone) + select { + case <-workCtx.Done(): + _ = cfg.Session.Close() + closeInput(cfg.InitialReader) + closeInput(cfg.Stdin) + case <-stopWatch: + } + }() + defer func() { close(stopWatch); <-watchDone }() + receiveDone := make(chan error, 1) + go func() { + err := receiveDatagrams(workCtx, cfg) + cancel() + receiveDone <- err + }() + + var sendErr error + if cfg.InitialPayloadSet { + sendErr = sendDatagram(cfg.Session, cfg.InitialPayload) + } + if sendErr == nil && cfg.InitialReader != nil { + data, err := core.ReadAllLimited(cfg.InitialReader, core.MaxCompositeMaterialization, "WebTransport initial datagram") + closeInput(cfg.InitialReader) + if err != nil { + sendErr = err + } else { + sendErr = sendDatagram(cfg.Session, data) + } + } + if sendErr == nil && cfg.Stdin != nil { + sendErr = sendInputDatagrams(workCtx, cfg.Session, cfg.Stdin, cfg.DatagramMode) + closeInput(cfg.Stdin) + } + if sendErr != nil { + cancel() + _ = cfg.Session.Close() + <-receiveDone + return sendErr + } + return normalizeDatagramClose(ctx, <-receiveDone) +} + +func receiveDatagrams(ctx context.Context, cfg Config) error { + seq := uint64(0) + for { + data, err := cfg.Session.ReceiveDatagram(ctx) + if err != nil { + return err + } + record, _ := json.Marshal(struct { + Sequence uint64 `json:"sequence"` + Length int `json:"length"` + Data string `json:"data"` + }{seq, len(data), base64.StdEncoding.EncodeToString(data)}) + record = append(record, '\n') + if _, err := cfg.Stdout.Write(record); err != nil { + _ = cfg.Session.Close() + return err + } + seq++ + } +} + +func isCleanClose(err error) bool { + if err == nil || errors.Is(err, io.EOF) { + return true + } + var streamErr *webtransport.StreamError + if errors.As(err, &streamErr) && streamErr.ErrorCode == 0 { + return true + } + var sessionErr *webtransport.SessionError + return errors.As(err, &sessionErr) && sessionErr.ErrorCode == 0 +} + +func normalizeDatagramClose(parent context.Context, err error) error { + if err == nil || errors.Is(err, io.EOF) { + return nil + } + if parent.Err() != nil { + return context.Cause(parent) + } + var sessionErr *webtransport.SessionError + if errors.As(err, &sessionErr) && sessionErr.ErrorCode == 0 { + return nil + } + return err +} + +func sendDatagram(s Session, data []byte) error { + if err := s.SendDatagram(data); err != nil { + return fmt.Errorf("send WebTransport datagram (%d bytes): %w", len(data), err) + } + return nil +} + +func sendInputDatagrams(ctx context.Context, s Session, r io.Reader, mode core.WTDatagramMode) error { + if mode == core.WTDatagramBinary { + buf := make([]byte, core.MaxWebTransportBinaryChunk) + for { + n, err := r.Read(buf) + if n > 0 { + if e := sendDatagram(s, append([]byte(nil), buf[:n]...)); e != nil { + return e + } + } + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("read WebTransport datagram input: %w", err) + } + } + } + br := bufio.NewReaderSize(r, int(core.MaxWebTransportDatagramLine)+1) + for { + line, err := readLine(br) + if len(line) > int(core.MaxWebTransportDatagramLine) { + return core.LimitError{Subsystem: "WebTransport datagram line", Limit: core.MaxWebTransportDatagramLine} + } + if len(line) > 0 || err != io.EOF { + if e := sendDatagram(s, line); e != nil { + return e + } + } + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("read WebTransport datagram input: %w", err) + } + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + } +} + +func readLine(r *bufio.Reader) ([]byte, error) { + line := make([]byte, 0, 128) + for { + part, err := r.ReadSlice('\n') + line = append(line, part...) + if len(line) > int(core.MaxWebTransportDatagramLine)+1 { + return line, nil + } + if err != bufio.ErrBufferFull { + if len(line) > 0 && line[len(line)-1] == '\n' { + line = line[:len(line)-1] + } + return line, err + } + } +} + +func closeInput(r io.Reader) { + if c, ok := r.(io.Closer); ok { + _ = c.Close() + } +} + +// terminalWriter holds incomplete UTF-8 until the next read. All controls are +// escaped, so an attacker cannot create a terminal control sequence. +type terminalWriter struct { + dst io.Writer + pending []byte +} + +func (w *terminalWriter) Write(p []byte) (int, error) { + w.pending = append(w.pending, p...) + cut := incompleteUTF8Start(w.pending) + if cut < 0 { + cut = len(w.pending) + } + if cut == 0 { + return len(p), nil + } + out := core.AppendTerminalSafeBytes(nil, w.pending[:cut]) + if _, err := w.dst.Write(out); err != nil { + return 0, err + } + w.pending = append(w.pending[:0], w.pending[cut:]...) + return len(p), nil +} +func incompleteUTF8Start(p []byte) int { + for i := len(p) - 1; i >= 0 && i >= len(p)-4; i-- { + if !utf8.RuneStart(p[i]) { + continue + } + if !utf8.FullRune(p[i:]) { + return i + } + break + } + return -1 +} + +func (w *terminalWriter) Flush() error { + if len(w.pending) == 0 { + return nil + } + out := core.AppendTerminalSafeBytes(nil, w.pending) + w.pending = nil + _, err := w.dst.Write(out) + return err +} diff --git a/internal/wt/wt_test.go b/internal/wt/wt_test.go new file mode 100644 index 00000000..6391e7f8 --- /dev/null +++ b/internal/wt/wt_test.go @@ -0,0 +1,100 @@ +package wt + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/ryanfowler/fetch/internal/core" +) + +type fakeSession struct { + stream *fakeStream + datagrams [][]byte + received [][]byte +} + +func (s *fakeSession) OpenStream(context.Context) (io.ReadWriteCloser, error) { return s.stream, nil } +func (s *fakeSession) SendDatagram(p []byte) error { + s.datagrams = append(s.datagrams, append([]byte(nil), p...)) + return nil +} +func (s *fakeSession) ReceiveDatagram(context.Context) ([]byte, error) { + if len(s.received) == 0 { + return nil, errors.New("done") + } + p := s.received[0] + s.received = s.received[1:] + return p, nil +} +func (s *fakeSession) Close() error { return nil } + +type fakeStream struct { + bytes.Buffer + input []byte + closed bool +} + +func (s *fakeStream) Read(p []byte) (int, error) { + if len(s.input) == 0 { + return 0, io.EOF + } + n := copy(p, s.input) + s.input = s.input[n:] + return n, nil +} +func (s *fakeStream) Close() error { s.closed = true; return nil } + +func TestRunStreamDefersInputAndReadsAfterEOF(t *testing.T) { + s := &fakeSession{stream: &fakeStream{input: []byte("reply")}} + var out bytes.Buffer + err := Run(context.Background(), Config{Session: s, Stdout: &out, InitialReader: strings.NewReader("first"), Stdin: strings.NewReader("second")}) + if err != nil { + t.Fatal(err) + } + if got := s.stream.String(); got != "firstsecond" { + t.Fatalf("sent %q", got) + } + if out.String() != "reply" { + t.Fatalf("received %q", out.String()) + } + if !s.stream.closed { + t.Fatal("stream was not closed for writing") + } +} + +func TestRunDatagramsUsesOneInitialPayloadAndJSONLines(t *testing.T) { + s := &fakeSession{received: [][]byte{{0, 1}, {}}} + var out bytes.Buffer + err := Run(context.Background(), Config{Session: s, Stdout: &out, Mode: core.WTDatagram, InitialReader: strings.NewReader("a\nb"), Stdin: strings.NewReader("one\ntwo\n")}) + if err == nil { + t.Fatal("expected receive loop error") + } + if len(s.datagrams) != 3 || string(s.datagrams[0]) != "a\nb" || string(s.datagrams[1]) != "one" || string(s.datagrams[2]) != "two" { + t.Fatalf("datagrams %#v", s.datagrams) + } + want := "{\"sequence\":0,\"length\":2,\"data\":\"AAE=\"}\n{\"sequence\":1,\"length\":0,\"data\":\"\"}\n" + if out.String() != want { + t.Fatalf("output %q, want %q", out.String(), want) + } +} + +func TestTerminalWriterEscapesControlsAndKeepsUTF8(t *testing.T) { + var out bytes.Buffer + w := &terminalWriter{dst: &out} + if _, err := w.Write([]byte{0xe2}); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte{0x82, 0xac, 0x1b}); err != nil { + t.Fatal(err) + } + if err := w.Flush(); err != nil { + t.Fatal(err) + } + if out.String() != "€\\x1b" { + t.Fatalf("output %q", out.String()) + } +} diff --git a/main.go b/main.go index 69eee2d7..0b9aece7 100644 --- a/main.go +++ b/main.go @@ -99,7 +99,18 @@ func main() { os.Exit(1) } handle := core.NewHandle(app.Cfg.Color) - if err := app.Cfg.Validate(); err != nil { + // WebTransport preflight includes proxy selection and must run before + // certificate loading, update work, DNS, or stdin access. + if err := cli.ValidateWebTransport(app); err != nil { + p := handle.Stderr() + core.WriteErrorMsg(p, err) + os.Exit(1) + } + validateConfig := app.Cfg.Validate + if app.WebTransport { + validateConfig = app.Cfg.ValidateForWebTransport + } + if err := validateConfig(); err != nil { p := handle.Stderr() core.WriteErrorMsg(p, err) os.Exit(1) @@ -225,6 +236,7 @@ func main() { Image: app.Cfg.Image, Insecure: getValue(app.Cfg.Insecure), Method: app.Method, + MethodExplicit: app.OptionWasExplicit("method"), Multipart: multipart.NewMultipart(app.Multipart), NoEncode: getValue(app.Cfg.NoEncode), NoPager: app.Cfg.Pager == core.PagerUnknown && getValue(app.Cfg.NoPager), @@ -253,6 +265,10 @@ func main() { URL: app.URL, Verbosity: verbosity, WS: app.WS, + WebTransport: app.WebTransport, + WTMode: app.WTMode, + WTDgramMode: app.WTDgramMode, + WTProtocols: app.WTProtocols, WSInteractive: app.WSInteractive, WSMessageMode: app.WSMessageMode, SchemelessURL: app.SchemelessURL, @@ -289,6 +305,7 @@ func handleSkillCommand(ctx context.Context, app *cli.App) int { {Path: "references/grpc.md", Data: mustReadEmbeddedSkill("skills/fetch/references/grpc.md")}, {Path: "references/http.md", Data: mustReadEmbeddedSkill("skills/fetch/references/http.md")}, {Path: "references/websocket.md", Data: mustReadEmbeddedSkill("skills/fetch/references/websocket.md")}, + {Path: "references/webtransport.md", Data: mustReadEmbeddedSkill("skills/fetch/references/webtransport.md")}, {Path: "evals/evals.json", Data: mustReadEmbeddedSkill("skills/fetch/evals/evals.json")}, } bundle, err := skill.NewBundle(core.Version, files) diff --git a/skills/fetch/SKILL.md b/skills/fetch/SKILL.md index 9f6bf4f3..a637ee29 100644 --- a/skills/fetch/SKILL.md +++ b/skills/fetch/SKILL.md @@ -3,7 +3,7 @@ name: fetch description: > Use the fetch CLI to call and debug HTTP APIs, inspect JSON responses, test authentication, diagnose DNS and TLS, measure request timing, extract - readable articles, call gRPC services, and interact with WebSockets. Prefer + readable articles, call gRPC services, and interact with WebSockets or WebTransport. Prefer this skill when a task requires making or troubleshooting a network request from the terminal. license: MIT @@ -16,7 +16,7 @@ metadata: # fetch Use `fetch` for terminal-native HTTP, API, article extraction, DNS/TLS, gRPC, -and WebSocket work. +WebSocket, and WebTransport work. ## Agent-safe defaults @@ -78,8 +78,8 @@ fetch --grpc -j @request.json URL/SERVICE/METHOD ``` Read [HTTP recipes](references/http.md), [diagnostics](references/diagnostics.md), -[gRPC](references/grpc.md), or [WebSockets](references/websocket.md) only when the -task needs that detail. +[gRPC](references/grpc.md), [WebSockets](references/websocket.md), or +[WebTransport](references/webtransport.md) only when the task needs that detail. ## Article extraction @@ -107,7 +107,7 @@ streaming calls. Use `--http 1`, `--http 2`, or `--http 3` to force a protocol. Automatic HTTP/3 is opportunistic for direct HTTPS and never sends the request twice. `--ech auto` uses Encrypted ClientHello when DNS advertises a valid configuration; -`--ech on` requires accepted ECH and cannot be combined with forced HTTP/3. +`--ech on` requires accepted ECH for ordinary HTTP/3. WebTransport supports mandatory ECH through its HTTP/3 dial path. ECH is not used through proxies or cleartext HTTP/2. Inspect DNS and TLS before changing trust or transport settings. The repository's `docs/limits.md` describes the shared body and protocol caps. @@ -125,6 +125,13 @@ WebSocket text lines, binary messages, and interactive entries are limited to handshake and continues receiving until the peer closes. Use `--ws-message-mode text|binary|auto` when message type matters. +WebTransport uses `--webtransport https://HOST/PATH`. It requires direct HTTP/3 +UDP and does not use WebSocket flags or message types. Stream mode sends raw +bytes on one reliable bidirectional stream. Datagram mode uses +`--wt-datagram-mode lines|binary`; received datagrams are base64 JSON Lines +records. Datagram input EOF does not close the session, so cancellation may be +required. + ## Security - Never invent or print credentials. Prefer existing environment variables, diff --git a/skills/fetch/evals/evals.json b/skills/fetch/evals/evals.json index 92a68f41..50970b9f 100644 --- a/skills/fetch/evals/evals.json +++ b/skills/fetch/evals/evals.json @@ -30,6 +30,26 @@ "The call uses --grpc and -j @request.json", "The method URL contains package.Inventory/GetItem" ] + }, + { + "id": 4, + "prompt": "Connect to an HTTPS WebTransport endpoint and receive datagrams that a script can parse.", + "expected_output": "Use HTTP/3 WebTransport datagram mode and keep stdout as JSON Lines.", + "assertions": [ + "The command includes --webtransport and --wt-mode datagram", + "The response does not recommend WebSocket-only flags", + "The response mentions JSON Lines datagram output" + ] + }, + { + "id": 5, + "prompt": "Send two lines over a WebTransport stream and make sure binary response bytes are not formatted.", + "expected_output": "Use stream mode with direct stdin and explain raw redirected output and cancellation behavior.", + "assertions": [ + "The command uses --webtransport with stream mode", + "The response distinguishes streams from datagrams", + "The response mentions cancellation or peer closure" + ] } ] } diff --git a/skills/fetch/references/webtransport.md b/skills/fetch/references/webtransport.md new file mode 100644 index 00000000..575cdf4e --- /dev/null +++ b/skills/fetch/references/webtransport.md @@ -0,0 +1,18 @@ +## WebTransport + +Use `--webtransport` with an `https://` URL. It selects HTTP/3 over direct +UDP; do not use WebSocket flags or WebSocket message semantics. + +The default `--wt-mode stream` opens one reliable bidirectional stream. `-d`, +`-j`, and piped input are raw bytes sent after the handshake. Stream output is +raw when redirected and terminal-safe when displayed. + +`--wt-mode datagram` sends datagrams. `--wt-datagram-mode lines` sends one +line per datagram, and `binary` sends 1 KiB chunks. Received datagrams are +compact JSON Lines records containing `sequence`, `length`, and base64 `data`. +Input EOF does not close a datagram session; cancellation or peer closure does. + +Repeat `--wt-protocol` for application protocol offers. WebTransport v1 does +not support proxies, redirects, retries, output files, `--format`, HAR, or +Digest authentication. `--dry-run` is network-free and does not consume +stdin.