From 40575980dfb8e769f95f334256c8ec85931da3b4 Mon Sep 17 00:00:00 2001 From: Zihan Dai <99155080+PDGGK@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:39:24 +1000 Subject: [PATCH 1/2] fix(services/vercel-blob): send the pagination cursor so listing terminates The lister stores the cursor the server returns: ctx.done = !resp.has_more; if let Some(cursor) = resp.cursor { ctx.token = cursor; } and nothing ever reads it back. VercelBlobCore::list took only a prefix and a limit, so page two was byte-identical to page one: https://blob.vercel-storage.com?prefix={prefix}&limit={limit} ctx.token is the only write of that field in the crate. Meanwhile PageLister::next loops while ctx.done is false, so a listing the server truncates never advances: the same page is fetched and pushed again for as long as has_more stays true. Operator::list collects into a Vec and grows without bound; lister() streams the same entries forever. Vercel documents cursor beside prefix and limit -- "a string obtained from a previous list response to be used for reading the next page of results" -- and the default limit is 1000, so any prefix holding more than that reaches this without the caller setting anything. list now takes the cursor and appends it when non-empty, and the URL construction moves into build_list_url so it can be asserted directly. The four internal lookups pass "": each asks for Some(1) to resolve a single blob's URL and never paginates. One extra guard in the lister: has_more true with no cursor now ends the listing rather than leaving ctx.done false. That truncates, but the only other option is re-requesting the same page indefinitely. Four unit tests. Dropping the cursor from the URL again fails exactly the two that involve one; the first-page and root-prefix tests hold either way. --- core/services/vercel-blob/src/core.rs | 89 +++++++++++++++++++++---- core/services/vercel-blob/src/lister.rs | 12 +++- 2 files changed, 84 insertions(+), 17 deletions(-) diff --git a/core/services/vercel-blob/src/core.rs b/core/services/vercel-blob/src/core.rs index 7b8dfd8802a3..779489780bf9 100644 --- a/core/services/vercel-blob/src/core.rs +++ b/core/services/vercel-blob/src/core.rs @@ -96,7 +96,7 @@ impl VercelBlobCore { let p = build_abs_path(&self.root, path); // Vercel blob use an unguessable random id url to download the file // So we use list to get the url of the file and then use it to download the file - let resp = self.list(ctx, &p, Some(1)).await?; + let resp = self.list(ctx, &p, Some(1), "").await?; // Use the mtach url to download the file let url = resolve_blob(resp.blobs, p); @@ -161,7 +161,7 @@ impl VercelBlobCore { pub async fn head(&self, ctx: &OperationContext, path: &str) -> Result> { let p = build_abs_path(&self.root, path); - let resp = self.list(ctx, &p, Some(1)).await?; + let resp = self.list(ctx, &p, Some(1), "").await?; let url = resolve_blob(resp.blobs, p); @@ -194,7 +194,7 @@ impl VercelBlobCore { ) -> Result> { let from = build_abs_path(&self.root, from); - let resp = self.list(ctx, &from, Some(1)).await?; + let resp = self.list(ctx, &from, Some(1), "").await?; let from_url = resolve_blob(resp.blobs, from); @@ -229,17 +229,9 @@ impl VercelBlobCore { ctx: &OperationContext, prefix: &str, limit: Option, + cursor: &str, ) -> Result { - let prefix = if prefix == "/" { "" } else { prefix }; - - let mut url = format!( - "https://blob.vercel-storage.com?prefix={}", - percent_encode_path(prefix) - ); - - if let Some(limit) = limit { - url.push_str(&format!("&limit={limit}")) - } + let url = build_list_url(prefix, limit, cursor); let req = Request::get(&url); @@ -271,7 +263,7 @@ impl VercelBlobCore { pub async fn vercel_delete_blob(&self, ctx: &OperationContext, path: &str) -> Result<()> { let p = build_abs_path(&self.root, path); - let resp = self.list(ctx, &p, Some(1)).await?; + let resp = self.list(ctx, &p, Some(1), "").await?; let url = resolve_blob(resp.blobs, p); @@ -410,6 +402,31 @@ impl VercelBlobCore { } } +/// Build the `list` request URL. +/// +/// `cursor` is the token the previous page returned; Vercel documents it alongside `prefix` and +/// `limit` as "a string obtained from a previous list response to be used for reading the next +/// page of results". Without it every page request is identical to the first, and a truncated +/// listing never advances. +fn build_list_url(prefix: &str, limit: Option, cursor: &str) -> String { + let prefix = if prefix == "/" { "" } else { prefix }; + + let mut url = format!( + "https://blob.vercel-storage.com?prefix={}", + percent_encode_path(prefix) + ); + + if let Some(limit) = limit { + url.push_str(&format!("&limit={limit}")) + } + + if !cursor.is_empty() { + url.push_str(&format!("&cursor={}", percent_encode_path(cursor))); + } + + url +} + pub fn parse_blob(blob: &Blob) -> Result { let mode = if blob.pathname.ends_with('/') { EntryMode::DIR @@ -572,3 +589,47 @@ mod error { } pub(super) use error::*; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_first_page_carries_no_cursor() { + let url = build_list_url("dir/", Some(100), ""); + + assert_eq!(url, "https://blob.vercel-storage.com?prefix=dir/&limit=100"); + } + + #[test] + fn a_later_page_resumes_from_the_cursor() { + // Without this parameter the second request is byte-identical to the first, so a + // truncated listing re-emits the same page for as long as `hasMore` stays true. + let url = build_list_url("dir/", Some(100), "eyJvZmZzZXQiOjEwMH0"); + + assert_eq!( + url, + "https://blob.vercel-storage.com?prefix=dir/&limit=100&cursor=eyJvZmZzZXQiOjEwMH0" + ); + } + + #[test] + fn the_cursor_is_percent_encoded() { + // The cursor is an opaque server token; an unescaped `&` or `=` would silently graft + // extra parameters onto the query. + let url = build_list_url("", None, "a&b=c d"); + + assert_eq!( + url, + "https://blob.vercel-storage.com?prefix=&cursor=a%26b%3Dc%20d" + ); + } + + #[test] + fn the_root_prefix_is_sent_empty() { + assert_eq!( + build_list_url("/", None, ""), + "https://blob.vercel-storage.com?prefix=" + ); + } +} diff --git a/core/services/vercel-blob/src/lister.rs b/core/services/vercel-blob/src/lister.rs index 8ae238eac811..4931012fe48a 100644 --- a/core/services/vercel-blob/src/lister.rs +++ b/core/services/vercel-blob/src/lister.rs @@ -52,12 +52,18 @@ impl oio::PageList for VercelBlobLister { async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> { let p = build_abs_path(&self.core.root, &self.path); - let resp = self.core.list(&self.ctx, &p, self.limit).await?; + let resp = self + .core + .list(&self.ctx, &p, self.limit, &ctx.token) + .await?; ctx.done = !resp.has_more; - if let Some(cursor) = resp.cursor { - ctx.token = cursor; + match resp.cursor { + Some(cursor) => ctx.token = cursor, + // More pages were reported but nothing was handed back to resume from. Stopping + // truncates the listing; the alternative is re-requesting the same page forever. + None => ctx.done = true, } for blob in resp.blobs { From 7f1812c18a0088e86116d5fe45a06b2fb5a858e9 Mon Sep 17 00:00:00 2001 From: Zihan Dai <99155080+PDGGK@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:49:51 +1000 Subject: [PATCH 2/2] Address review: Option<&str> cursor, inline the URL, drop the tests Review points from @erickguan, all three applied: - cursor is Option<&str> rather than an empty string standing in for "no cursor"; the four single-blob lookups pass None, and the lister turns an empty ctx.token into None - build_list_url removed, the format! goes back inline - test module removed The functional change is unchanged: the cursor is sent when there is one, percent-encoded because it is an opaque server token. --- core/services/vercel-blob/src/core.rs | 94 +++++-------------------- core/services/vercel-blob/src/lister.rs | 11 +-- 2 files changed, 26 insertions(+), 79 deletions(-) diff --git a/core/services/vercel-blob/src/core.rs b/core/services/vercel-blob/src/core.rs index 779489780bf9..e662455c97ca 100644 --- a/core/services/vercel-blob/src/core.rs +++ b/core/services/vercel-blob/src/core.rs @@ -96,7 +96,7 @@ impl VercelBlobCore { let p = build_abs_path(&self.root, path); // Vercel blob use an unguessable random id url to download the file // So we use list to get the url of the file and then use it to download the file - let resp = self.list(ctx, &p, Some(1), "").await?; + let resp = self.list(ctx, &p, Some(1), None).await?; // Use the mtach url to download the file let url = resolve_blob(resp.blobs, p); @@ -161,7 +161,7 @@ impl VercelBlobCore { pub async fn head(&self, ctx: &OperationContext, path: &str) -> Result> { let p = build_abs_path(&self.root, path); - let resp = self.list(ctx, &p, Some(1), "").await?; + let resp = self.list(ctx, &p, Some(1), None).await?; let url = resolve_blob(resp.blobs, p); @@ -194,7 +194,7 @@ impl VercelBlobCore { ) -> Result> { let from = build_abs_path(&self.root, from); - let resp = self.list(ctx, &from, Some(1), "").await?; + let resp = self.list(ctx, &from, Some(1), None).await?; let from_url = resolve_blob(resp.blobs, from); @@ -229,9 +229,22 @@ impl VercelBlobCore { ctx: &OperationContext, prefix: &str, limit: Option, - cursor: &str, + cursor: Option<&str>, ) -> Result { - let url = build_list_url(prefix, limit, cursor); + let prefix = if prefix == "/" { "" } else { prefix }; + + let mut url = format!( + "https://blob.vercel-storage.com?prefix={}", + percent_encode_path(prefix) + ); + + if let Some(limit) = limit { + url.push_str(&format!("&limit={limit}")) + } + + if let Some(cursor) = cursor { + url.push_str(&format!("&cursor={}", percent_encode_path(cursor))); + } let req = Request::get(&url); @@ -263,7 +276,7 @@ impl VercelBlobCore { pub async fn vercel_delete_blob(&self, ctx: &OperationContext, path: &str) -> Result<()> { let p = build_abs_path(&self.root, path); - let resp = self.list(ctx, &p, Some(1), "").await?; + let resp = self.list(ctx, &p, Some(1), None).await?; let url = resolve_blob(resp.blobs, p); @@ -402,31 +415,6 @@ impl VercelBlobCore { } } -/// Build the `list` request URL. -/// -/// `cursor` is the token the previous page returned; Vercel documents it alongside `prefix` and -/// `limit` as "a string obtained from a previous list response to be used for reading the next -/// page of results". Without it every page request is identical to the first, and a truncated -/// listing never advances. -fn build_list_url(prefix: &str, limit: Option, cursor: &str) -> String { - let prefix = if prefix == "/" { "" } else { prefix }; - - let mut url = format!( - "https://blob.vercel-storage.com?prefix={}", - percent_encode_path(prefix) - ); - - if let Some(limit) = limit { - url.push_str(&format!("&limit={limit}")) - } - - if !cursor.is_empty() { - url.push_str(&format!("&cursor={}", percent_encode_path(cursor))); - } - - url -} - pub fn parse_blob(blob: &Blob) -> Result { let mode = if blob.pathname.ends_with('/') { EntryMode::DIR @@ -589,47 +577,3 @@ mod error { } pub(super) use error::*; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn the_first_page_carries_no_cursor() { - let url = build_list_url("dir/", Some(100), ""); - - assert_eq!(url, "https://blob.vercel-storage.com?prefix=dir/&limit=100"); - } - - #[test] - fn a_later_page_resumes_from_the_cursor() { - // Without this parameter the second request is byte-identical to the first, so a - // truncated listing re-emits the same page for as long as `hasMore` stays true. - let url = build_list_url("dir/", Some(100), "eyJvZmZzZXQiOjEwMH0"); - - assert_eq!( - url, - "https://blob.vercel-storage.com?prefix=dir/&limit=100&cursor=eyJvZmZzZXQiOjEwMH0" - ); - } - - #[test] - fn the_cursor_is_percent_encoded() { - // The cursor is an opaque server token; an unescaped `&` or `=` would silently graft - // extra parameters onto the query. - let url = build_list_url("", None, "a&b=c d"); - - assert_eq!( - url, - "https://blob.vercel-storage.com?prefix=&cursor=a%26b%3Dc%20d" - ); - } - - #[test] - fn the_root_prefix_is_sent_empty() { - assert_eq!( - build_list_url("/", None, ""), - "https://blob.vercel-storage.com?prefix=" - ); - } -} diff --git a/core/services/vercel-blob/src/lister.rs b/core/services/vercel-blob/src/lister.rs index 4931012fe48a..5ca0ac9478ee 100644 --- a/core/services/vercel-blob/src/lister.rs +++ b/core/services/vercel-blob/src/lister.rs @@ -52,10 +52,13 @@ impl oio::PageList for VercelBlobLister { async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> { let p = build_abs_path(&self.core.root, &self.path); - let resp = self - .core - .list(&self.ctx, &p, self.limit, &ctx.token) - .await?; + let cursor = if ctx.token.is_empty() { + None + } else { + Some(ctx.token.as_str()) + }; + + let resp = self.core.list(&self.ctx, &p, self.limit, cursor).await?; ctx.done = !resp.has_more;