diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 07ed26655..7398e461a 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,8 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: output: fixtures merged: true @@ -47,7 +48,8 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/test@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +86,8 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: output: fixtures merged: true @@ -114,7 +117,8 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/test@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +156,8 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: output: fixtures merged: true @@ -182,7 +187,8 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.13 + # TODO: unreleased gateway-conformance with IPIP-548 tests (ipfs/gateway-conformance#301); switch back to a release tag once one ships + uses: ipfs/gateway-conformance/.github/actions/test@713bb9d95f5cc5e81fa1f16a55235ecfd8fff50a with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8afb79da0..69c456ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,12 @@ The following emojis are used to highlight certain changes: ### Added +- ✨ `gateway`: responses now include the `Ipfs-Uri` header with a canonical `ipfs://` or `ipns://` URI for the requested content path, and expose it via the default `Access-Control-Expose-Headers`. The header carries the content root in canonical form (base32 CIDv1 for `/ipfs/`, base36 CIDv1 for cryptographic `/ipns/` names, lowercase FQDN for DNSLink) with percent-encoded path segments, so clients get a value that is safe in HTTP field context regardless of bytes in the underlying path. [IPIP-548](https://github.com/ipfs/specs/pull/548) + ### Changed +- 🛠 `gateway`: the deprecated `X-Ipfs-Path` response header is no longer sent by default; its value cannot represent all UnixFS file names and it is superseded by `Ipfs-Uri`. **Action required:** consumers that read `X-Ipfs-Path` should migrate to `Ipfs-Uri`; to restore the legacy header meanwhile, set `Config.DeprecatedXIpfsPath` and call `Headers.WithDeprecatedXIpfsPath` before `Headers.ApplyCors` so it is listed in `Access-Control-Expose-Headers` again. Even with the flag set, the header is omitted for content paths with bytes that cannot appear in an HTTP field value (Section 5.5 of RFC 9110), such as raw non-ASCII UnixFS file names: gateway-conformance fails a gateway that sends such values, and only `Ipfs-Uri` carries those paths. [IPIP-548](https://github.com/ipfs/specs/pull/548) + ### Removed ### Fixed diff --git a/gateway/gateway.go b/gateway/gateway.go index 9909aee52..1e56d8487 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -79,6 +79,21 @@ type Config struct { // is being proxied by other service, which wants to use the error message. DisableHTMLErrors bool + // DeprecatedXIpfsPath configures the gateway to send the legacy + // X-Ipfs-Path response header, deprecated by [IPIP-0548]. Disabled by + // default: the legacy value cannot represent all UnixFS file names (raw + // non-ASCII bytes are mangled per RFC 9110 field-value rules) and it is + // superseded by the Ipfs-Uri header, which is sent whenever the content + // root can be normalized. Enable only for backward compatibility with + // legacy consumers that still expect X-Ipfs-Path. When enabled, pair it + // with [Headers.WithDeprecatedXIpfsPath] so the header is also listed in + // Access-Control-Expose-Headers. Even when enabled, the header is + // omitted for content paths that contain bytes that cannot appear in an + // HTTP field value (Section 5.5 of RFC 9110), per [IPIP-0548]. + // + // [IPIP-0548]: https://github.com/ipfs/specs/pull/548 + DeprecatedXIpfsPath bool + // PublicGateways configures the behavior of known public gateways. Each key is // a fully qualified domain name (FQDN). To be used with WithHostname. PublicGateways map[string]*PublicGateway diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 30b519527..6a9b532bf 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -380,6 +380,101 @@ func TestHeaders(t *testing.T) { test(dagCborResponseFormat, dagCborPath, dagCborRoots) }) + t.Run("Ipfs-Uri contains expected values", func(t *testing.T) { + test := func(responseFormat string, path string, uri string) { + t.Run(responseFormat+" "+path, func(t *testing.T) { + url := ts.URL + path + req := mustNewRequest(t, http.MethodGet, url, nil) + req.Header.Add("Accept", responseFormat) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Equal(t, uri, res.Header.Get("Ipfs-Uri")) + }) + } + + test("", filePath, "ipfs://"+rootCID+"/subdir/fnord") + test("text/html", dirPath, "ipfs://"+rootCID+"/subdir/") + test("text/html", "/ipfs/"+rootCID+"/", "ipfs://"+rootCID+"/") + test("text/html", hamtFilePath, "ipfs://"+rootCID+"/hamt/685.txt") + test(dagJsonResponseFormat, dagCborPath, "ipfs://"+rootCID+"/subdir/dag-cbor-document") + }) + + t.Run("Ipfs-Uri is sent with 412 from Cache-Control: only-if-cached", func(t *testing.T) { + // Valid CID that is missing from the fixture, so only-if-cached + // returns 412 before the response format is known. + missingCID := "bafkreicm2cerwpdtah2rd7rxg5jcaqsj52blfuaprkurromr5y6p3a5zlu" + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+missingCID, nil) + req.Header.Add("Cache-Control", "only-if-cached") + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusPreconditionFailed, res.StatusCode) + require.Equal(t, "ipfs://"+missingCID, res.Header.Get("Ipfs-Uri")) + }) + + t.Run("Ipfs-Uri is sent with 400 from malformed Accept header", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil) + req.Header.Add("Accept", "application/vnd.ipld.car; version=1; version=2") + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusBadRequest, res.StatusCode) + require.Equal(t, "ipfs://"+rootCID+"/subdir/fnord", res.Header.Get("Ipfs-Uri")) + }) + + t.Run("X-Ipfs-Path is not sent by default", func(t *testing.T) { + req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Empty(t, res.Header.Get("X-Ipfs-Path")) + require.NotContains(t, res.Header.Values("Access-Control-Expose-Headers"), "X-Ipfs-Path") + require.Contains(t, res.Header.Values("Access-Control-Expose-Headers"), "Ipfs-Uri") + }) + + t.Run("X-Ipfs-Path is sent when DeprecatedXIpfsPath is enabled", func(t *testing.T) { + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + DeprecatedXIpfsPath: true, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+filePath, nil) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Equal(t, filePath, res.Header.Get("X-Ipfs-Path")) + require.Equal(t, "ipfs://"+rootCID+"/subdir/fnord", res.Header.Get("Ipfs-Uri")) + require.Contains(t, res.Header.Values("Access-Control-Expose-Headers"), "X-Ipfs-Path") + }) + + t.Run("X-Ipfs-Path is omitted for field-value-unsafe paths even when DeprecatedXIpfsPath is enabled", func(t *testing.T) { + // The content path /ipfs/{cid}/łódź.txt contains raw non-ASCII + // bytes, which cannot appear in an HTTP field value (Section 5.5 + // of RFC 9110), so the legacy header must be omitted despite the + // flag; Ipfs-Uri carries the percent-encoded path instead. + backend, root := newMockBackend(t, "dir-with-tricky-filenames.car") + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + DeprecatedXIpfsPath: true, + }) + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+root.String()+"/%C5%82%C3%B3d%C5%BA.txt", nil) + res := mustDoWithoutRedirect(t, req) + _, err := io.Copy(io.Discard, res.Body) + require.NoError(t, err) + defer res.Body.Close() + require.Equal(t, http.StatusOK, res.StatusCode) + require.Empty(t, res.Header.Get("X-Ipfs-Path")) + require.Equal(t, "ipfs://"+root.String()+"/%C5%82%C3%B3d%C5%BA.txt", res.Header.Get("Ipfs-Uri")) + }) + t.Run("If-None-Match with wrong value forces path resolution, but X-Ipfs-Roots is correct (regression)", func(t *testing.T) { test := func(responseFormat string, path string, roots string) { t.Run(responseFormat, func(t *testing.T) { diff --git a/gateway/handler.go b/gateway/handler.go index cbf14b698..eb7224871 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -28,6 +28,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "golang.org/x/net/idna" ) var log = logging.Logger("boxo/gateway") @@ -36,6 +37,11 @@ const ( ipfsPathPrefix = "/ipfs/" ipnsPathPrefix = ipns.NamespacePrefix immutableCacheControl = "public, max-age=29030400, immutable" + + // maxIpfsUriLength caps the value of the Ipfs-Uri response header. + // Longer values are not sent (IPIP-0548 allows omission) to stay under + // common per-field limits in reverse proxies. + maxIpfsUriLength = 8192 ) var ( @@ -261,6 +267,16 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } }() + // Set as soon as the content path is known, so redirect and error + // responses below carry it too (IPIP-0548). The content path does not + // change after this point. + if i.config.DeprecatedXIpfsPath && isFieldValueSafe(contentPath.String()) { + w.Header().Set("X-Ipfs-Path", contentPath.String()) + } + if uri, ok := ipfsUriHeaderValue(contentPath); ok && len(uri) <= maxIpfsUriLength { + w.Header().Set("Ipfs-Uri", uri) + } + if i.handleOnlyIfCached(w, r, contentPath) { return } @@ -274,8 +290,6 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat)) i.requestTypeMetric.WithLabelValues(contentPath.Namespace(), responseFormat).Inc() - w.Header().Set("X-Ipfs-Path", contentPath.String()) - // Fail fast if unsupported request type was sent to a Trustless Gateway. if !i.isDeserializedResponsePossible(r) && !i.isTrustlessRequest(contentPath, responseFormat) { err := errors.New("only trustless requests are accepted on this gateway: https://specs.ipfs.tech/http-gateways/trustless-gateway/") @@ -545,6 +559,123 @@ func setIpfsRootsHeader(w http.ResponseWriter, rq *requestData, md *ContentPathM w.Header().Set("X-Ipfs-Roots", rootCidList) } +// isFieldValueSafe reports whether s can be carried byte-for-byte in an HTTP +// field value: only HTAB (0x09), SP (0x20), and visible ASCII (0x21-0x7E) +// are allowed (Section 5.5 of RFC 9110). CR, LF, and NUL are rejected or +// replaced by HTTP software, other control characters are invalid, and +// non-ASCII bytes arrive garbled, so the legacy X-Ipfs-Path header MUST be +// omitted for such content paths (IPIP-0548); Ipfs-Uri percent-encodes them +// instead. +func isFieldValueSafe(s string) bool { + for i := 0; i < len(s); i++ { + if c := s[i]; c != '\t' && (c < 0x20 || c > 0x7E) { + return false + } + } + return true +} + +// ipfsUriHeaderValue builds the value of the Ipfs-Uri response header for the +// given content path (IPIP-0548): an ipfs:// or ipns:// URI with the content +// root in its canonical text form, and a URI path that mirrors the content +// path remainder (everything after "/{namespace}/{root}") with every segment +// percent-encoded, so a trailing slash is kept. It returns false when the +// namespace has no URI scheme or the content root cannot be normalized, in +// which case the header is not sent. +func ipfsUriHeaderValue(contentPath path.Path) (string, bool) { + segments := contentPath.Segments() + if len(segments) < 2 { + return "", false + } + + var authority string + switch contentPath.Namespace() { + case path.IPFSNamespace: + // Canonical form: CIDv1 in lowercase base32. + c, err := cid.Decode(segments[1]) + if err != nil { + return "", false + } + authority, err = cid.NewCidV1(c.Type(), c.Hash()).StringOfBase(multibase.Base32) + if err != nil { + return "", false + } + case path.IPNSNamespace: + if name, err := ipns.NameFromString(segments[1]); err == nil { + // Canonical form for cryptographic IPNS names: CIDv1 in + // lowercase base36, preserving the multicodec. ipns.Name + // covers libp2p-key, the only codec in use today; roots with + // other codecs fall through to the DNSLink branch, fail its + // checks, and the header is omitted, as IPIP-0548 allows. + authority = name.String() + } else { + // DNSLink: canonical form is the lowercase FQDN as A-labels, + // without the optional trailing dot. Omit the header when the + // root does not convert to A-labels or the result is not a + // multi-label DNS name. + host, err := idna.Lookup.ToASCII(strings.TrimSuffix(segments[1], ".")) + if err != nil || !strings.Contains(host, ".") || + strings.Contains(host, "..") || strings.HasPrefix(host, ".") || strings.HasSuffix(host, ".") { + return "", false + } + authority = host + } + default: + return "", false + } + + var b strings.Builder + b.WriteString(contentPath.Namespace()) + b.WriteString("://") + b.WriteString(authority) + // The URI path mirrors the content path remainder: split on "/", each + // segment percent-encoded, rejoined with "/". The content path is already + // normalized (path.NewPath collapses duplicate slashes and dot segments + // but keeps a trailing slash), so the remainder here is "", "/", or + // "/segment..." with an optional trailing slash, all of which the header + // value reproduces. + remainder := strings.TrimPrefix(contentPath.String(), "/"+segments[0]+"/"+segments[1]) + if remainder != "" { + for _, segment := range strings.Split(remainder[1:], "/") { + b.WriteByte('/') + b.WriteString(encodeIpfsUriSegment(segment)) + } + } + return b.String(), true +} + +// encodeIpfsUriSegment percent-encodes a single content path segment for use +// in an Ipfs-Uri value: every byte outside the RFC 3986 unreserved set becomes +// %XX with uppercase hex, so the output is ASCII-only and byte-identical +// across implementations. This is stricter than url.PathEscape, which leaves +// sub-delims like "&" and "=" unencoded. A segment that is exactly "." or +// ".." is emitted fully percent-encoded (IPIP-0548), so it cannot be taken +// for a relative dot segment; such segments never survive the gateway's +// request path normalization, but the encoding is total for any input. +func encodeIpfsUriSegment(segment string) string { + switch segment { + case ".": + return "%2E" + case "..": + return "%2E%2E" + } + const upperhex = "0123456789ABCDEF" + var b strings.Builder + b.Grow(len(segment)) + for i := 0; i < len(segment); i++ { + c := segment[i] + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || + c == '-' || c == '.' || c == '_' || c == '~' { + b.WriteByte(c) + } else { + b.WriteByte('%') + b.WriteByte(upperhex[c>>4]) + b.WriteByte(upperhex[c&0xF]) + } + } + return b.String() +} + // lastModifiedMatch returns true if we can respond with HTTP 304 Not Modified // It compares If-Modified-Since with logical modification time read from DAG // (e.g. UnixFS 1.5 modtime, if present) diff --git a/gateway/handler_test.go b/gateway/handler_test.go index 3045ee721..3dbe76a9f 100644 --- a/gateway/handler_test.go +++ b/gateway/handler_test.go @@ -3,9 +3,117 @@ package gateway import ( "testing" + "github.com/ipfs/boxo/path" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestIpfsUriHeaderValue(t *testing.T) { + const ( + // CIDv1 already in the canonical lowercase base32 form. + cidV1 = "bafkreiba3vpkcqpc6xtp3hsatzcod6iwneouzjoq7ymy4m2js6gc3czt6i" + // The same DAG root in CIDv0 and canonical CIDv1 base32 form. + cidV0 = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG" + cidV0AsBase32 = "bafybeie5nqv6kd3qnfjupgvz34woh3oksc3iau6abmyajn7qvtf6d2ho34" + // The same ed25519 IPNS name as a base58 peer ID and as the + // canonical base36 CIDv1 with libp2p-key codec. + peerIDBase58 = "12D3KooWA4Xop1JaT3MHxwYMkCepYsv4iPVopMXwCz5iHYdBfeSB" + ipnsNameB36 = "k51qzi5uqu5dg9ufswxt229ntzdy7p4125xzv5rtyjso89ajdujg6csfxcj260" + ) + + for _, test := range []struct { + name string + contentPath string + expected string + ok bool + }{ + // Path segment percent-encoding (IPIP-0548 shared test vectors). + {"unreserved characters kept as-is", "/ipfs/" + cidV1 + "/plain.txt", "ipfs://" + cidV1 + "/plain.txt", true}, + {"space", "/ipfs/" + cidV1 + "/with space.txt", "ipfs://" + cidV1 + "/with%20space.txt", true}, + {"percent sign", "/ipfs/" + cidV1 + "/100% sure.txt", "ipfs://" + cidV1 + "/100%25%20sure.txt", true}, + {"hash and question mark", "/ipfs/" + cidV1 + "/a#b?c.txt", "ipfs://" + cidV1 + "/a%23b%3Fc.txt", true}, + {"name that already looks percent-encoded", "/ipfs/" + cidV1 + "/Portugal%2C+España=Peninsula Ibérica.txt", "ipfs://" + cidV1 + "/Portugal%252C%2BEspa%C3%B1a%3DPeninsula%20Ib%C3%A9rica.txt", true}, + {"multibyte utf-8", "/ipfs/" + cidV1 + "/łódź.txt", "ipfs://" + cidV1 + "/%C5%82%C3%B3d%C5%BA.txt", true}, + {"emoji", "/ipfs/" + cidV1 + "/emoji🚀.txt", "ipfs://" + cidV1 + "/emoji%F0%9F%9A%80.txt", true}, + // Filenames in major scripts (IPIP-0548 shared test vectors). + {"chinese", "/ipfs/" + cidV1 + "/你好.txt", "ipfs://" + cidV1 + "/%E4%BD%A0%E5%A5%BD.txt", true}, + {"japanese", "/ipfs/" + cidV1 + "/ファイル.txt", "ipfs://" + cidV1 + "/%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB.txt", true}, + {"korean", "/ipfs/" + cidV1 + "/파일.txt", "ipfs://" + cidV1 + "/%ED%8C%8C%EC%9D%BC.txt", true}, + {"arabic", "/ipfs/" + cidV1 + "/ملف.txt", "ipfs://" + cidV1 + "/%D9%85%D9%84%D9%81.txt", true}, + {"hebrew", "/ipfs/" + cidV1 + "/קובץ.txt", "ipfs://" + cidV1 + "/%D7%A7%D7%95%D7%91%D7%A5.txt", true}, + {"cyrillic", "/ipfs/" + cidV1 + "/файл.txt", "ipfs://" + cidV1 + "/%D1%84%D0%B0%D0%B9%D0%BB.txt", true}, + {"greek", "/ipfs/" + cidV1 + "/αρχείο.txt", "ipfs://" + cidV1 + "/%CE%B1%CF%81%CF%87%CE%B5%CE%AF%CE%BF.txt", true}, + {"devanagari", "/ipfs/" + cidV1 + "/नमस्ते.txt", "ipfs://" + cidV1 + "/%E0%A4%A8%E0%A4%AE%E0%A4%B8%E0%A5%8D%E0%A4%A4%E0%A5%87.txt", true}, + {"thai", "/ipfs/" + cidV1 + "/ไฟล์.txt", "ipfs://" + cidV1 + "/%E0%B9%84%E0%B8%9F%E0%B8%A5%E0%B9%8C.txt", true}, + {"multiple segments", "/ipfs/" + cidV1 + "/subdir/with space.txt", "ipfs://" + cidV1 + "/subdir/with%20space.txt", true}, + // Path mirroring: the URI path reproduces the content path remainder, + // including a trailing slash. Dot segments and duplicate slashes are + // collapsed by path.NewPath, mirroring gateway request normalization. + {"trailing slash on directory kept", "/ipfs/" + cidV1 + "/subdir/", "ipfs://" + cidV1 + "/subdir/", true}, + {"root only with trailing slash", "/ipfs/" + cidV1 + "/", "ipfs://" + cidV1 + "/", true}, + {"single dot segment collapsed", "/ipfs/" + cidV1 + "/a/./b", "ipfs://" + cidV1 + "/a/b", true}, + {"double dot segment collapsed", "/ipfs/" + cidV1 + "/a/../b", "ipfs://" + cidV1 + "/b", true}, + {"duplicate slashes collapsed", "/ipfs/" + cidV1 + "//a//b", "ipfs://" + cidV1 + "/a/b", true}, + // Authority normalization. + {"cidv0 root normalized to base32 cidv1", "/ipfs/" + cidV0 + "/łódź.txt", "ipfs://" + cidV0AsBase32 + "/%C5%82%C3%B3d%C5%BA.txt", true}, + {"root only", "/ipfs/" + cidV1, "ipfs://" + cidV1, true}, + {"ipns peer id normalized to base36 cidv1", "/ipns/" + peerIDBase58, "ipns://" + ipnsNameB36, true}, + {"ipns name already canonical", "/ipns/" + ipnsNameB36 + "/sub", "ipns://" + ipnsNameB36 + "/sub", true}, + {"dnslink", "/ipns/en.wikipedia-on-ipfs.org/wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, + {"dnslink lowercased", "/ipns/EN.WIKIPEDIA-ON-IPFS.ORG/wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, + {"dnslink trailing dot stripped", "/ipns/en.wikipedia-on-ipfs.org./wiki", "ipns://en.wikipedia-on-ipfs.org/wiki", true}, + {"dnslink unicode converted to a-labels", "/ipns/ŻÓŁĆ.example.net/wiki", "ipns://xn--kda4b0koi.example.net/wiki", true}, + {"dnslink on a private network", "/ipns/example.local/wiki", "ipns://example.local/wiki", true}, + // Roots that cannot be normalized: header is omitted. + {"ipld namespace has no uri scheme", "/ipld/" + cidV1, "", false}, + {"ipns root neither name nor dnslink", "/ipns/notavalidname", "", false}, + {"dnslink with no dot", "/ipns/examplemissingtld", "", false}, + {"dnslink with empty label", "/ipns/en..example.net/wiki", "", false}, + } { + t.Run(test.name, func(t *testing.T) { + contentPath, err := path.NewPath(test.contentPath) + require.NoError(t, err) + value, ok := ipfsUriHeaderValue(contentPath) + assert.Equal(t, test.ok, ok) + assert.Equal(t, test.expected, value) + }) + } +} + +func TestEncodeIpfsUriSegment(t *testing.T) { + // Dot segments never survive path.NewPath, so the encoder is tested + // directly: IPIP-0548 requires them fully percent-encoded. + assert.Equal(t, "%2E", encodeIpfsUriSegment(".")) + assert.Equal(t, "%2E%2E", encodeIpfsUriSegment("..")) + // Dots elsewhere are unreserved and stay as-is. + assert.Equal(t, "...a", encodeIpfsUriSegment("...a")) + assert.Equal(t, "a.b", encodeIpfsUriSegment("a.b")) +} + +func TestIsFieldValueSafe(t *testing.T) { + for _, test := range []struct { + name string + in string + safe bool + }{ + {"empty", "", true}, + {"visible ascii", "/ipfs/bafkreiba3vpkcqpc6xtp3hsatzcod6iwneouzjoq7ymy4m2js6gc3czt6i/file.txt", true}, + {"htab", "a\tb", true}, + {"space", "a b", true}, + {"boundary 0x21 and 0x7e", "!~", true}, + {"cr", "a\rb", false}, + {"lf", "a\nb", false}, + {"nul", "a\x00b", false}, + {"del 0x7f", "a\x7fb", false}, + {"other control 0x1b", "a\x1bb", false}, + {"utf-8 multibyte", "łódź.txt", false}, + } { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.safe, isFieldValueSafe(test.in)) + }) + } +} + func TestEtagMatch(t *testing.T) { for _, test := range []struct { header string // value in If-None-Match HTTP header diff --git a/gateway/headers.go b/gateway/headers.go index 691ba9858..308d05c09 100644 --- a/gateway/headers.go +++ b/gateway/headers.go @@ -8,7 +8,8 @@ import ( // Headers is an HTTP middleware that sets the configured headers in all requests. type Headers struct { - headers map[string][]string + headers map[string][]string + exposeXIpfsPath bool } // NewHeaders creates a new [Headers] middleware that applies the given headers @@ -26,6 +27,15 @@ func NewHeaders(headers map[string][]string) *Headers { return h } +// WithDeprecatedXIpfsPath includes the legacy X-Ipfs-Path header in the +// default Access-Control-Expose-Headers set added by [Headers.ApplyCors]. +// Call it before [Headers.ApplyCors], and only when the gateway is +// configured to send the header (see [Config.DeprecatedXIpfsPath]). +func (h *Headers) WithDeprecatedXIpfsPath() *Headers { + h.exposeXIpfsPath = true + return h +} + // ApplyCors applies safe default HTTP headers for controlling cross-origin // requests. This function adds several values to the [Access-Control-Allow-Headers] // and [Access-Control-Expose-Headers] entries to be exposed on GET and OPTIONS @@ -70,15 +80,18 @@ func (h *Headers) ApplyCors() *Headers { "X-Requested-With", }, h.headers[ACAHeadersName]...)) - h.headers[ACEHeadersName] = cleanHeaderSet( - append([]string{ - "Content-Length", - "Content-Range", - "X-Chunked-Output", - "X-Stream-Output", - "X-Ipfs-Path", - "X-Ipfs-Roots", - }, h.headers[ACEHeadersName]...)) + exposeHeaders := []string{ + "Content-Length", + "Content-Range", + "X-Chunked-Output", + "X-Stream-Output", + "Ipfs-Uri", + "X-Ipfs-Roots", + } + if h.exposeXIpfsPath { + exposeHeaders = append(exposeHeaders, "X-Ipfs-Path") + } + h.headers[ACEHeadersName] = cleanHeaderSet(append(exposeHeaders, h.headers[ACEHeadersName]...)) return h } diff --git a/gateway/testdata/dir-with-tricky-filenames.car b/gateway/testdata/dir-with-tricky-filenames.car new file mode 100644 index 000000000..33d9b616e Binary files /dev/null and b/gateway/testdata/dir-with-tricky-filenames.car differ diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index b7743ef7a..d72c8e777 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -253,7 +253,11 @@ func newTestServerWithConfigAndHeaders(t *testing.T, backend IPFSBackend, config mux.Handle("/ipfs/", handler) mux.Handle("/ipns/", handler) handler = NewHostnameHandler(config, backend, mux) - handler = NewHeaders(headers).ApplyCors().Wrap(handler) + corsHeaders := NewHeaders(headers) + if config.DeprecatedXIpfsPath { + corsHeaders = corsHeaders.WithDeprecatedXIpfsPath() + } + handler = corsHeaders.ApplyCors().Wrap(handler) ts := httptest.NewServer(handler) t.Cleanup(func() { ts.Close() }) diff --git a/go.mod b/go.mod index e3136070a..9f98b7b76 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 go.uber.org/zap v1.28.0 golang.org/x/exp v0.0.0-20260718201538-764159d718ef + golang.org/x/net v0.57.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -157,7 +158,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.54.0 // indirect golang.org/x/mod v0.38.0 // indirect - golang.org/x/net v0.57.0 // indirect golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75 // indirect golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect