From 37cbbddf894db87578462a91251db45ffc65fdab Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 21 Aug 2026 04:12:20 +0200 Subject: [PATCH 1/4] feat(gateway): Ipfs-Uri response header Implement the Ipfs-Uri response header from ipfs/specs#548: a canonical ipfs:// or ipns:// URI for the requested content path, safe in an HTTP field value for any UnixFS file name. - authority in canonical form: base32 CIDv1, base36 IPNS name keeping its multicodec, lowercase A-label DNSLink; header omitted when the root cannot be normalized - every path segment percent-encoded over the RFC 3986 unreserved set with uppercase hex; "." and ".." emitted as %2E / %2E%2E - sent on deserialized, redirect, and error responses up to 8192 bytes, and exposed via default Access-Control-Expose-Headers - X-Ipfs-Path no longer sent by default; restore via Config.DeprecatedXIpfsPath + Headers.WithDeprecatedXIpfsPath, still omitted for bytes invalid in a field value - CI runs gateway-conformance from ipfs/gateway-conformance#301 until a release with the IPIP-548 tests ships --- .github/workflows/gateway-conformance.yml | 18 ++- CHANGELOG.md | 4 + gateway/gateway.go | 15 ++ gateway/gateway_test.go | 95 ++++++++++++ gateway/handler.go | 135 +++++++++++++++++- gateway/handler_test.go | 106 ++++++++++++++ gateway/headers.go | 33 +++-- .../testdata/dir-with-tricky-filenames.car | Bin 0 -> 1782 bytes gateway/utilities_test.go | 6 +- go.mod | 2 +- 10 files changed, 394 insertions(+), 20 deletions(-) create mode 100644 gateway/testdata/dir-with-tricky-filenames.car diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 07ed26655..0bcb601db 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,8 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@149f396ec8a229293bf7e96df3185d2ce4a93dcf with: output: fixtures merged: true @@ -47,7 +48,8 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/test@149f396ec8a229293bf7e96df3185d2ce4a93dcf with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +86,8 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@149f396ec8a229293bf7e96df3185d2ce4a93dcf with: output: fixtures merged: true @@ -114,7 +117,8 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/test@149f396ec8a229293bf7e96df3185d2ce4a93dcf with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +156,8 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@149f396ec8a229293bf7e96df3185d2ce4a93dcf with: output: fixtures merged: true @@ -182,7 +187,8 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/test@149f396ec8a229293bf7e96df3185d2ce4a93dcf with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8afb79da0..69c456ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,12 @@ The following emojis are used to highlight certain changes: ### Added +- ✨ `gateway`: responses now include the `Ipfs-Uri` header with a canonical `ipfs://` or `ipns://` URI for the requested content path, and expose it via the default `Access-Control-Expose-Headers`. The header carries the content root in canonical form (base32 CIDv1 for `/ipfs/`, base36 CIDv1 for cryptographic `/ipns/` names, lowercase FQDN for DNSLink) with percent-encoded path segments, so clients get a value that is safe in HTTP field context regardless of bytes in the underlying path. [IPIP-548](https://github.com/ipfs/specs/pull/548) + ### Changed +- 🛠 `gateway`: the deprecated `X-Ipfs-Path` response header is no longer sent by default; its value cannot represent all UnixFS file names and it is superseded by `Ipfs-Uri`. **Action required:** consumers that read `X-Ipfs-Path` should migrate to `Ipfs-Uri`; to restore the legacy header meanwhile, set `Config.DeprecatedXIpfsPath` and call `Headers.WithDeprecatedXIpfsPath` before `Headers.ApplyCors` so it is listed in `Access-Control-Expose-Headers` again. Even with the flag set, the header is omitted for content paths with bytes that cannot appear in an HTTP field value (Section 5.5 of RFC 9110), such as raw non-ASCII UnixFS file names: gateway-conformance fails a gateway that sends such values, and only `Ipfs-Uri` carries those paths. [IPIP-548](https://github.com/ipfs/specs/pull/548) + ### Removed ### Fixed diff --git a/gateway/gateway.go b/gateway/gateway.go index 9909aee52..1e56d8487 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -79,6 +79,21 @@ type Config struct { // is being proxied by other service, which wants to use the error message. DisableHTMLErrors bool + // DeprecatedXIpfsPath configures the gateway to send the legacy + // X-Ipfs-Path response header, deprecated by [IPIP-0548]. Disabled by + // default: the legacy value cannot represent all UnixFS file names (raw + // non-ASCII bytes are mangled per RFC 9110 field-value rules) and it is + // superseded by the Ipfs-Uri header, which is sent whenever the content + // root can be normalized. Enable only for backward compatibility with + // legacy consumers that still expect X-Ipfs-Path. When enabled, pair it + // with [Headers.WithDeprecatedXIpfsPath] so the header is also listed in + // Access-Control-Expose-Headers. Even when enabled, the header is + // omitted for content paths that contain bytes that cannot appear in an + // HTTP field value (Section 5.5 of RFC 9110), per [IPIP-0548]. + // + // [IPIP-0548]: https://github.com/ipfs/specs/pull/548 + DeprecatedXIpfsPath bool + // PublicGateways configures the behavior of known public gateways. Each key is // a fully qualified domain name (FQDN). To be used with WithHostname. PublicGateways map[string]*PublicGateway diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 30b519527..6a9b532bf 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -380,6 +380,101 @@ func TestHeaders(t *testing.T) { test(dagCborResponseFormat, dagCborPath, dagCborRoots) }) + t.Run("Ipfs-Uri contains expected values", func(t *testing.T) { + test := func(responseFormat string, path string, uri string) { + t.Run(responseFormat+" "+path, func(t *testing.T) { + url := ts.URL + path + req := mustNewRequest(t, http.MethodGet, url, nil) + req.Header.Add("Accept", responseFormat) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Equal(t, uri, res.Header.Get("Ipfs-Uri")) + }) + } + + test("", filePath, "ipfs://"+rootCID+"/subdir/fnord") + test("text/html", dirPath, "ipfs://"+rootCID+"/subdir/") + test("text/html", "/ipfs/"+rootCID+"/", "ipfs://"+rootCID+"/") + test("text/html", hamtFilePath, "ipfs://"+rootCID+"/hamt/685.txt") + test(dagJsonResponseFormat, dagCborPath, "ipfs://"+rootCID+"/subdir/dag-cbor-document") + }) + + t.Run("Ipfs-Uri is sent with 412 from Cache-Control: only-if-cached", func(t *testing.T) { + // Valid CID that is missing from the fixture, so only-if-cached + // returns 412 before the response format is known. + missingCID := "bafkreicm2cerwpdtah2rd7rxg5jcaqsj52blfuaprkurromr5y6p3a5zlu" + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+missingCID, nil) + req.Header.Add("Cache-Control", "only-if-cached") + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusPreconditionFailed, res.StatusCode) + require.Equal(t, "ipfs://"+missingCID, res.Header.Get("Ipfs-Uri")) + }) + + t.Run("Ipfs-Uri is sent with 400 from malformed Accept header", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil) + req.Header.Add("Accept", "application/vnd.ipld.car; version=1; version=2") + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusBadRequest, res.StatusCode) + require.Equal(t, "ipfs://"+rootCID+"/subdir/fnord", res.Header.Get("Ipfs-Uri")) + }) + + t.Run("X-Ipfs-Path is not sent by default", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Empty(t, res.Header.Get("X-Ipfs-Path")) + require.NotContains(t, res.Header.Values("Access-Control-Expose-Headers"), "X-Ipfs-Path") + require.Contains(t, res.Header.Values("Access-Control-Expose-Headers"), "Ipfs-Uri") + }) + + t.Run("X-Ipfs-Path is sent when DeprecatedXIpfsPath is enabled", func(t *testing.T) { + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + DeprecatedXIpfsPath: true, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Equal(t, filePath, res.Header.Get("X-Ipfs-Path")) + require.Equal(t, "ipfs://"+rootCID+"/subdir/fnord", res.Header.Get("Ipfs-Uri")) + require.Contains(t, res.Header.Values("Access-Control-Expose-Headers"), "X-Ipfs-Path") + }) + + t.Run("X-Ipfs-Path is omitted for field-value-unsafe paths even when DeprecatedXIpfsPath is enabled", func(t *testing.T) { + // The content path /ipfs/{cid}/łódź.txt contains raw non-ASCII + // bytes, which cannot appear in an HTTP field value (Section 5.5 + // of RFC 9110), so the legacy header must be omitted despite the + // flag; Ipfs-Uri carries the percent-encoded path instead. + backend, root := newMockBackend(t, "dir-with-tricky-filenames.car") + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + DeprecatedXIpfsPath: true, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/%C5%82%C3%B3d%C5%BA.txt", nil) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Empty(t, res.Header.Get("X-Ipfs-Path")) + require.Equal(t, "ipfs://"+root.String()+"/%C5%82%C3%B3d%C5%BA.txt", res.Header.Get("Ipfs-Uri")) + }) + t.Run("If-None-Match with wrong value forces path resolution, but X-Ipfs-Roots is correct (regression)", func(t *testing.T) { test := func(responseFormat string, path string, roots string) { t.Run(responseFormat, func(t *testing.T) { diff --git a/gateway/handler.go b/gateway/handler.go index cbf14b698..eb7224871 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -28,6 +28,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "golang.org/x/net/idna" ) var log = logging.Logger("boxo/gateway") @@ -36,6 +37,11 @@ const ( ipfsPathPrefix = "/ipfs/" ipnsPathPrefix = ipns.NamespacePrefix immutableCacheControl = "public, max-age=29030400, immutable" + + // maxIpfsUriLength caps the value of the Ipfs-Uri response header. + // Longer values are not sent (IPIP-0548 allows omission) to stay under + // common per-field limits in reverse proxies. + maxIpfsUriLength = 8192 ) var ( @@ -261,6 +267,16 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } }() + // Set as soon as the content path is known, so redirect and error + // responses below carry it too (IPIP-0548). The content path does not + // change after this point. + if i.config.DeprecatedXIpfsPath && isFieldValueSafe(contentPath.String()) { + w.Header().Set("X-Ipfs-Path", contentPath.String()) + } + if uri, ok := ipfsUriHeaderValue(contentPath); ok && len(uri) <= maxIpfsUriLength { + w.Header().Set("Ipfs-Uri", uri) + } + if i.handleOnlyIfCached(w, r, contentPath) { return } @@ -274,8 +290,6 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat)) i.requestTypeMetric.WithLabelValues(contentPath.Namespace(), responseFormat).Inc() - w.Header().Set("X-Ipfs-Path", contentPath.String()) - // Fail fast if unsupported request type was sent to a Trustless Gateway. if !i.isDeserializedResponsePossible(r) && !i.isTrustlessRequest(contentPath, responseFormat) { err := errors.New("only trustless requests are accepted on this gateway: https://specs.ipfs.tech/http-gateways/trustless-gateway/") @@ -545,6 +559,123 @@ func setIpfsRootsHeader(w http.ResponseWriter, rq *requestData, md *ContentPathM w.Header().Set("X-Ipfs-Roots", rootCidList) } +// isFieldValueSafe reports whether s can be carried byte-for-byte in an HTTP +// field value: only HTAB (0x09), SP (0x20), and visible ASCII (0x21-0x7E) +// are allowed (Section 5.5 of RFC 9110). CR, LF, and NUL are rejected or +// replaced by HTTP software, other control characters are invalid, and +// non-ASCII bytes arrive garbled, so the legacy X-Ipfs-Path header MUST be +// omitted for such content paths (IPIP-0548); Ipfs-Uri percent-encodes them +// instead. +func isFieldValueSafe(s string) bool { + for i := 0; i < len(s); i++ { + if c := s[i]; c != '\t' && (c < 0x20 || c > 0x7E) { + return false + } + } + return true +} + +// ipfsUriHeaderValue builds the value of the Ipfs-Uri response header for the +// given content path (IPIP-0548): an ipfs:// or ipns:// URI with the content +// root in its canonical text form, and a URI path that mirrors the content +// path remainder (everything after "/{namespace}/{root}") with every segment +// percent-encoded, so a trailing slash is kept. It returns false when the +// namespace has no URI scheme or the content root cannot be normalized, in +// which case the header is not sent. +func ipfsUriHeaderValue(contentPath path.Path) (string, bool) { + segments := contentPath.Segments() + if len(segments) < 2 { + return "", false + } + + var authority string + switch contentPath.Namespace() { + case path.IPFSNamespace: + // Canonical form: CIDv1 in lowercase base32. + c, err := cid.Decode(segments[1]) + if err != nil { + return "", false + } + authority, err = cid.NewCidV1(c.Type(), c.Hash()).StringOfBase(multibase.Base32) + if err != nil { + return "", false + } + case path.IPNSNamespace: + if name, err := ipns.NameFromString(segments[1]); err == nil { + // Canonical form for cryptographic IPNS names: CIDv1 in + // lowercase base36, preserving the multicodec. ipns.Name + // covers libp2p-key, the only codec in use today; roots with + // other codecs fall through to the DNSLink branch, fail its + // checks, and the header is omitted, as IPIP-0548 allows. + authority = name.String() + } else { + // DNSLink: canonical form is the lowercase FQDN as A-labels, + // without the optional trailing dot. Omit the header when the + // root does not convert to A-labels or the result is not a + // multi-label DNS name. + host, err := idna.Lookup.ToASCII(strings.TrimSuffix(segments[1], ".")) + if err != nil || !strings.Contains(host, ".") || + strings.Contains(host, "..") || strings.HasPrefix(host, ".") || strings.HasSuffix(host, ".") { + return "", false + } + authority = host + } + default: + return "", false + } + + var b strings.Builder + b.WriteString(contentPath.Namespace()) + b.WriteString("://") + b.WriteString(authority) + // The URI path mirrors the content path remainder: split on "/", each + // segment percent-encoded, rejoined with "/". The content path is already + // normalized (path.NewPath collapses duplicate slashes and dot segments + // but keeps a trailing slash), so the remainder here is "", "/", or + // "/segment..." with an optional trailing slash, all of which the header + // value reproduces. + remainder := strings.TrimPrefix(contentPath.String(), "/"+segments[0]+"/"+segments[1]) + if remainder != "" { + for _, segment := range strings.Split(remainder[1:], "/") { + b.WriteByte('/') + b.WriteString(encodeIpfsUriSegment(segment)) + } + } + return b.String(), true +} + +// encodeIpfsUriSegment percent-encodes a single content path segment for use +// in an Ipfs-Uri value: every byte outside the RFC 3986 unreserved set becomes +// %XX with uppercase hex, so the output is ASCII-only and byte-identical +// across implementations. This is stricter than url.PathEscape, which leaves +// sub-delims like "&" and "=" unencoded. A segment that is exactly "." or +// ".." is emitted fully percent-encoded (IPIP-0548), so it cannot be taken +// for a relative dot segment; such segments never survive the gateway's +// request path normalization, but the encoding is total for any input. +func encodeIpfsUriSegment(segment string) string { + switch segment { + case ".": + return "%2E" + case "..": + return "%2E%2E" + } + const upperhex = "0123456789ABCDEF" + var b strings.Builder + b.Grow(len(segment)) + for i := 0; i < len(segment); i++ { + c := segment[i] + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || + c == '-' || c == '.' || c == '_' || c == '~' { + b.WriteByte(c) + } else { + b.WriteByte('%') + b.WriteByte(upperhex[c>>4]) + b.WriteByte(upperhex[c&0xF]) + } + } + return b.String() +} + // lastModifiedMatch returns true if we can respond with HTTP 304 Not Modified // It compares If-Modified-Since with logical modification time read from DAG // (e.g. UnixFS 1.5 modtime, if present) diff --git a/gateway/handler_test.go b/gateway/handler_test.go index 3045ee721..bc3823543 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -3,9 +3,115 @@ package gateway import ( "testing" + "github.com/ipfs/boxo/path" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestIpfsUriHeaderValue(t *testing.T) { + const ( + // CIDv1 already in the canonical lowercase base32 form. + cidV1 = "bafkreiba3vpkcqpc6xtp3hsatzcod6iwneouzjoq7ymy4m2js6gc3czt6i" + // The same DAG root in CIDv0 and canonical CIDv1 base32 form. + cidV0 = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG" + cidV0AsBase32 = "bafybeie5nqv6kd3qnfjupgvz34woh3oksc3iau6abmyajn7qvtf6d2ho34" + // The same ed25519 IPNS name as a base58 peer ID and as the + // canonical base36 CIDv1 with libp2p-key codec. + peerIDBase58 = "12D3KooWA4Xop1JaT3MHxwYMkCepYsv4iPVopMXwCz5iHYdBfeSB" + ipnsNameB36 = "k51qzi5uqu5dg9ufswxt229ntzdy7p4125xzv5rtyjso89ajdujg6csfxcj260" + ) + + for _, test := range []struct { + name string + contentPath string + expected string + ok bool + }{ + // Path segment percent-encoding (IPIP-0548 shared test vectors). + {"unreserved characters kept as-is", "/ipfs/" + cidV1 + "/plain.txt", "ipfs://" + cidV1 + "/plain.txt", true}, + {"space", "/ipfs/" + cidV1 + "/with space.txt", "ipfs://" + cidV1 + "/with%20space.txt", true}, + {"percent sign", "/ipfs/" + cidV1 + "/100% sure.txt", "ipfs://" + cidV1 + "/100%25%20sure.txt", true}, + {"hash and question mark", "/ipfs/" + cidV1 + "/a#b?c.txt", "ipfs://" + cidV1 + "/a%23b%3Fc.txt", true}, + {"name that already looks percent-encoded", "/ipfs/" + cidV1 + "/Portugal%2C+España=Peninsula Ibérica.txt", "ipfs://" + cidV1 + "/Portugal%252C%2BEspa%C3%B1a%3DPeninsula%20Ib%C3%A9rica.txt", true}, + {"multibyte utf-8", "/ipfs/" + cidV1 + "/łódź.txt", "ipfs://" + cidV1 + "/%C5%82%C3%B3d%C5%BA.txt", true}, + {"emoji", "/ipfs/" + cidV1 + "/emoji🚀.txt", "ipfs://" + cidV1 + "/emoji%F0%9F%9A%80.txt", true}, + // Filenames in major scripts (IPIP-0548 shared test vectors). + {"chinese", "/ipfs/" + cidV1 + "/你好.txt", "ipfs://" + cidV1 + "/%E4%BD%A0%E5%A5%BD.txt", true}, + {"japanese", "/ipfs/" + cidV1 + "/ファイル.txt", "ipfs://" + cidV1 + "/%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB.txt", true}, + {"korean", "/ipfs/" + cidV1 + "/파일.txt", "ipfs://" + cidV1 + "/%ED%8C%8C%EC%9D%BC.txt", true}, + {"arabic", "/ipfs/" + cidV1 + "/ملف.txt", "ipfs://" + cidV1 + "/%D9%85%D9%84%D9%81.txt", true}, + {"hebrew", "/ipfs/" + cidV1 + "/קובץ.txt", "ipfs://" + cidV1 + "/%D7%A7%D7%95%D7%91%D7%A5.txt", true}, + {"cyrillic", "/ipfs/" + cidV1 + "/файл.txt", "ipfs://" + cidV1 + "/%D1%84%D0%B0%D0%B9%D0%BB.txt", true}, + {"greek", "/ipfs/" + cidV1 + "/αρχείο.txt", "ipfs://" + cidV1 + "/%CE%B1%CF%81%CF%87%CE%B5%CE%AF%CE%BF.txt", true}, + {"devanagari", "/ipfs/" + cidV1 + "/नमस्ते.txt", "ipfs://" + cidV1 + "/%E0%A4%A8%E0%A4%AE%E0%A4%B8%E0%A5%8D%E0%A4%A4%E0%A5%87.txt", true}, + {"thai", "/ipfs/" + cidV1 + "/ไฟล์.txt", "ipfs://" + cidV1 + "/%E0%B9%84%E0%B8%9F%E0%B8%A5%E0%B9%8C.txt", true}, + {"multiple segments", "/ipfs/" + cidV1 + "/subdir/with space.txt", "ipfs://" + cidV1 + "/subdir/with%20space.txt", true}, + // Path mirroring: the URI path reproduces the content path remainder, + // including a trailing slash. Dot segments and duplicate slashes are + // collapsed by path.NewPath, mirroring gateway request normalization. + {"trailing slash on directory kept", "/ipfs/" + cidV1 + "/subdir/", "ipfs://" + cidV1 + "/subdir/", true}, + {"root only with trailing slash", "/ipfs/" + cidV1 + "/", "ipfs://" + cidV1 + "/", true}, + {"single dot segment collapsed", "/ipfs/" + cidV1 + "/a/./b", "ipfs://" + cidV1 + "/a/b", true}, + {"double dot segment collapsed", "/ipfs/" + cidV1 + "/a/../b", "ipfs://" + cidV1 + "/b", true}, + {"duplicate slashes collapsed", "/ipfs/" + cidV1 + "//a//b", "ipfs://" + cidV1 + "/a/b", true}, + // Authority normalization. + {"cidv0 root normalized to base32 cidv1", "/ipfs/" + cidV0 + "/łódź.txt", "ipfs://" + cidV0AsBase32 + "/%C5%82%C3%B3d%C5%BA.txt", true}, + {"root only", "/ipfs/" + cidV1, "ipfs://" + cidV1, true}, + {"ipns peer id normalized to base36 cidv1", "/ipns/" + peerIDBase58, "ipns://" + ipnsNameB36, true}, + {"ipns name already canonical", "/ipns/" + ipnsNameB36 + "/sub", "ipns://" + ipnsNameB36 + "/sub", true}, + {"dnslink", "/ipns/en.wikipedia-on-ipfs.org/wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, + {"dnslink lowercased", "/ipns/EN.WIKIPEDIA-ON-IPFS.ORG/wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, + {"dnslink trailing dot stripped", "/ipns/en.wikipedia-on-ipfs.org./wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, + {"dnslink unicode converted to a-labels", "/ipns/ŻÓŁĆ.example.net/wiki", "ipns://xn--kda4b0koi.example.net/wiki", true}, + // Roots that cannot be normalized: header is omitted. + {"ipld namespace has no uri scheme", "/ipld/" + cidV1, "", false}, + {"ipns root neither name nor dnslink", "/ipns/notavalidname", "", false}, + {"dnslink with empty label", "/ipns/en..example.net/wiki", "", false}, + } { + t.Run(test.name, func(t *testing.T) { + contentPath, err := path.NewPath(test.contentPath) + require.NoError(t, err) + value, ok := ipfsUriHeaderValue(contentPath) + assert.Equal(t, test.ok, ok) + assert.Equal(t, test.expected, value) + }) + } +} + +func TestEncodeIpfsUriSegment(t *testing.T) { + // Dot segments never survive path.NewPath, so the encoder is tested + // directly: IPIP-0548 requires them fully percent-encoded. + assert.Equal(t, "%2E", encodeIpfsUriSegment(".")) + assert.Equal(t, "%2E%2E", encodeIpfsUriSegment("..")) + // Dots elsewhere are unreserved and stay as-is. + assert.Equal(t, "...a", encodeIpfsUriSegment("...a")) + assert.Equal(t, "a.b", encodeIpfsUriSegment("a.b")) +} + +func TestIsFieldValueSafe(t *testing.T) { + for _, test := range []struct { + name string + in string + safe bool + }{ + {"empty", "", true}, + {"visible ascii", "/ipfs/bafkreiba3vpkcqpc6xtp3hsatzcod6iwneouzjoq7ymy4m2js6gc3czt6i/file.txt", true}, + {"htab", "a\tb", true}, + {"space", "a b", true}, + {"boundary 0x21 and 0x7e", "!~", true}, + {"cr", "a\rb", false}, + {"lf", "a\nb", false}, + {"nul", "a\x00b", false}, + {"del 0x7f", "a\x7fb", false}, + {"other control 0x1b", "a\x1bb", false}, + {"utf-8 multibyte", "łódź.txt", false}, + } { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.safe, isFieldValueSafe(test.in)) + }) + } +} + func TestEtagMatch(t *testing.T) { for _, test := range []struct { header string // value in If-None-Match HTTP header diff --git a/gateway/headers.go b/gateway/headers.go index 691ba9858..308d05c09 100644 --- a/gateway/headers.go +++ b/gateway/headers.go @@ -8,7 +8,8 @@ import ( // Headers is an HTTP middleware that sets the configured headers in all requests. type Headers struct { - headers map[string][]string + headers map[string][]string + exposeXIpfsPath bool } // NewHeaders creates a new [Headers] middleware that applies the given headers @@ -26,6 +27,15 @@ func NewHeaders(headers map[string][]string) *Headers { return h } +// WithDeprecatedXIpfsPath includes the legacy X-Ipfs-Path header in the +// default Access-Control-Expose-Headers set added by [Headers.ApplyCors]. +// Call it before [Headers.ApplyCors], and only when the gateway is +// configured to send the header (see [Config.DeprecatedXIpfsPath]). +func (h *Headers) WithDeprecatedXIpfsPath() *Headers { + h.exposeXIpfsPath = true + return h +} + // ApplyCors applies safe default HTTP headers for controlling cross-origin // requests. This function adds several values to the [Access-Control-Allow-Headers] // and [Access-Control-Expose-Headers] entries to be exposed on GET and OPTIONS @@ -70,15 +80,18 @@ func (h *Headers) ApplyCors() *Headers { "X-Requested-With", }, h.headers[ACAHeadersName]...)) - h.headers[ACEHeadersName] = cleanHeaderSet( - append([]string{ - "Content-Length", - "Content-Range", - "X-Chunked-Output", - "X-Stream-Output", - "X-Ipfs-Path", - "X-Ipfs-Roots", - }, h.headers[ACEHeadersName]...)) + exposeHeaders := []string{ + "Content-Length", + "Content-Range", + "X-Chunked-Output", + "X-Stream-Output", + "Ipfs-Uri", + "X-Ipfs-Roots", + } + if h.exposeXIpfsPath { + exposeHeaders = append(exposeHeaders, "X-Ipfs-Path") + } + h.headers[ACEHeadersName] = cleanHeaderSet(append(exposeHeaders, h.headers[ACEHeadersName]...)) return h } diff --git a/gateway/testdata/dir-with-tricky-filenames.car b/gateway/testdata/dir-with-tricky-filenames.car new file mode 100644 index 0000000000000000000000000000000000000000..33d9b616ebdcb5229e9fddbebd49a6fc4a378070 GIT binary patch literal 1782 zcma*meM}o=9KdmkX2J=yJTM@G=<)8D4$6yEK!I$S8pVMbNr@ZtD7W@Xxw7l=7I8o+ z3XRbl`w~P1nQvtOu?Z8c^dlews)7Ze$54#L=Hmga`<^pm%f_De#&fg|Fd884-g zFFS42;g(SNm8+es_C}O=HDmp3f`72}va)(~FH#!amGV1qMsrjOY_+Aal!gL!TIH-} z--AKjqm;bhyCeQSPd+#@o1z5*_w_?(W{#(sW`JX^62Wj9xdIXMihe%wV(73%*=R=&jK_n?8MipRo72nJF~P?fvX}`D3DwPN7dV3nX4aGN#z)g=;d+c!=h{zfg(8cKYqaP;s5{u literal 0 HcmV?d00001 diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index b7743ef7a..d72c8e777 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -253,7 +253,11 @@ func newTestServerWithConfigAndHeaders(t *testing.T, backend IPFSBackend, config mux.Handle("/ipfs/", handler) mux.Handle("/ipns/", handler) handler = NewHostnameHandler(config, backend, mux) - handler = NewHeaders(headers).ApplyCors().Wrap(handler) + corsHeaders := NewHeaders(headers) + if config.DeprecatedXIpfsPath { + corsHeaders = corsHeaders.WithDeprecatedXIpfsPath() + } + handler = corsHeaders.ApplyCors().Wrap(handler) ts := httptest.NewServer(handler) t.Cleanup(func() { ts.Close() }) diff --git a/go.mod b/go.mod index e3136070a..9f98b7b76 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20260718201538-764159d718ef + golang.org/x/net v0.57.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -157,7 +158,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.54.0 // indirect golang.org/x/mod v0.38.0 // indirect - golang.org/x/net v0.57.0 // indirect golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75 // indirect golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect From 558abb2ea72ca74cdeba68411126da6e7f262fca Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 24 Aug 2026 01:20:42 +0200 Subject: [PATCH 2/4] test: dotted vs dot-less DNSLink Ipfs-Uri Per IPIP-548 only dotted DNSLink names become an ipns:// authority: example.local works, examplenodot never does. Refs https://github.com/ipfs/helia-verified-fetch/pull/376#discussion_r3838134648 --- gateway/handler_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gateway/handler_test.go b/gateway/handler_test.go index bc3823543..5eeabb573 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -63,9 +63,11 @@ func TestIpfsUriHeaderValue(t *testing.T) { {"dnslink lowercased", "/ipns/EN.WIKIPEDIA-ON-IPFS.ORG/wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, {"dnslink trailing dot stripped", "/ipns/en.wikipedia-on-ipfs.org./wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, {"dnslink unicode converted to a-labels", "/ipns/ŻÓŁĆ.example.net/wiki", "ipns://xn--kda4b0koi.example.net/wiki", true}, + {"dnslink on a private network", "/ipns/example.local/wiki", "ipns://example.local/wiki", true}, // Roots that cannot be normalized: header is omitted. {"ipld namespace has no uri scheme", "/ipld/" + cidV1, "", false}, {"ipns root neither name nor dnslink", "/ipns/notavalidname", "", false}, + {"dnslink with no dot", "/ipns/examplenodot", "", false}, {"dnslink with empty label", "/ipns/en..example.net/wiki", "", false}, } { t.Run(test.name, func(t *testing.T) { From 919701685bd3da303b10121d40d603e7f1044ac5 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 24 Aug 2026 01:20:42 +0200 Subject: [PATCH 3/4] ci: bump gateway-conformance pin Use the conformance tests from ipfs/gateway-conformance#301 at its latest commit, which adds the dot-less DNSLink Ipfs-Uri tests. --- .github/workflows/gateway-conformance.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 0bcb601db..7398e461a 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -23,7 +23,7 @@ jobs: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@149f396ec8a229293bf7e96df3185d2ce4a93dcf + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: output: fixtures merged: true @@ -49,7 +49,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships - uses: ipfs/gateway-conformance/.github/actions/test@149f396ec8a229293bf7e96df3185d2ce4a93dcf + uses: ipfs/gateway-conformance/.github/actions/test@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -87,7 +87,7 @@ jobs: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@149f396ec8a229293bf7e96df3185d2ce4a93dcf + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: output: fixtures merged: true @@ -118,7 +118,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships - uses: ipfs/gateway-conformance/.github/actions/test@149f396ec8a229293bf7e96df3185d2ce4a93dcf + uses: ipfs/gateway-conformance/.github/actions/test@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -157,7 +157,7 @@ jobs: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@149f396ec8a229293bf7e96df3185d2ce4a93dcf + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: output: fixtures merged: true @@ -188,7 +188,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships - uses: ipfs/gateway-conformance/.github/actions/test@149f396ec8a229293bf7e96df3185d2ce4a93dcf + uses: ipfs/gateway-conformance/.github/actions/test@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 From 6f8fbc69153e3d7e9fffad93dfe6cf1cb5720e4d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 24 Aug 2026 02:16:30 +0200 Subject: [PATCH 4/4] test: rename dot-less name to examplemissingtld --- gateway/handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/handler_test.go b/gateway/handler_test.go index 5eeabb573..3dbe76a9f 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -67,7 +67,7 @@ func TestIpfsUriHeaderValue(t *testing.T) { // Roots that cannot be normalized: header is omitted. {"ipld namespace has no uri scheme", "/ipld/" + cidV1, "", false}, {"ipns root neither name nor dnslink", "/ipns/notavalidname", "", false}, - {"dnslink with no dot", "/ipns/examplenodot", "", false}, + {"dnslink with no dot", "/ipns/examplemissingtld", "", false}, {"dnslink with empty label", "/ipns/en..example.net/wiki", "", false}, } { t.Run(test.name, func(t *testing.T) {