From aeb82d4454daa9265a6c26c5c5e0dd0cf083f469 Mon Sep 17 00:00:00 2001 From: Toan Le Date: Mon, 8 Jun 2026 11:18:30 +0700 Subject: [PATCH 1/2] HADOOP-19973. Hadoop file system can not browse the folder that has children folder with colon in Windows. ABFS built list-status entry paths with File.separator, which is a backslash on Windows. Path("\name:x") then treats "\name" as a URI scheme, so listing a directory whose children contain a colon failed with URISyntaxException. The separator here is part of an ABFS/URI path, not a local filesystem path, so use a forward slash unconditionally. Contains content generated by Claude. Generated-by: Claude Opus 5 Co-authored-by: Claude Opus 5 (1M context) --- .../fs/azurebfs/services/AbfsClient.java | 3 +- .../fs/azurebfs/services/TestAbfsClient.java | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index a979b0aa1e8fdb..fe12902e9d4fa4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -19,7 +19,6 @@ package org.apache.hadoop.fs.azurebfs.services; import java.io.Closeable; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -1809,7 +1808,7 @@ protected VersionedFileStatus getVersionedFileStatusFromEntry( entry.lastModified()); } - Path entryPath = new Path(File.separator + entry.name()); + Path entryPath = new Path(AbfsHttpConstants.FORWARD_SLASH + entry.name()); if (uri != null) { entryPath = entryPath.makeQualified(uri, entryPath); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 8b0d7d728f562b..541612c4813305 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -29,8 +29,10 @@ import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; import org.apache.hadoop.fs.azurebfs.AbfsCountersImpl; import org.apache.hadoop.fs.azurebfs.MockIntercept; +import org.apache.hadoop.fs.azurebfs.contracts.services.DfsListResultEntrySchema; import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRICS_FORMAT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRICS_SHOULD_EMIT_ON_IDLE_TIME; import static org.apache.hadoop.fs.azurebfs.services.AbfsMetricsManager.ABFS_CLIENT_TIMER_THREAD_NAME; @@ -126,6 +128,47 @@ public void testTimerInitializationWithMetricCollection() throws Exception { .isEqualTo(false); } + /** + * Test that {@link AbfsClient#getVersionedFileStatusFromEntry} always builds the + * entry path using a forward slash, regardless of the platform-dependent + * {@link java.io.File#separator}. On Windows, {@code File.separator} is a + * backslash, and using it here used to break browsing of directories whose + * child entry names contain a colon (e.g. "dir:name"), because + * "\dir:name" gets misparsed as a URI with scheme "\dir". + */ + @Test + public void testGetVersionedFileStatusFromEntryUsesForwardSlash() throws Exception { + final Configuration configuration = new Configuration(); + AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration, ACCOUNT_NAME); + + AbfsCounters abfsCounters = spy(new AbfsCountersImpl(new URI("abcd"))); + AbfsClientContext abfsClientContext = new AbfsClientContextBuilder().withAbfsCounters(abfsCounters) + .withFileSystemId(UUID.randomUUID().toString()).build(); + + AbfsClient client = new AbfsDfsClient(new URL("https://" + ACCOUNT_NAME + "/"), + null, + abfsConfiguration, + (AccessTokenProvider) null, + null, + null, + abfsClientContext); + + final String entryName = "dir:withColon"; + DfsListResultEntrySchema entry = new DfsListResultEntrySchema() + .withName(entryName) + .withIsDirectory(true); + + VersionedFileStatus status = client.getVersionedFileStatusFromEntry(entry, null); + + assertThat(status.getPath().toUri().getPath()) + .describedAs("Entry path must be built with '/' regardless of the " + + "platform's File.separator, so folder names containing ':' " + + "are not misparsed as a URI scheme on Windows") + .isEqualTo(FORWARD_SLASH + entryName); + + client.close(); + } + /** * Check if a thread with the specified name is running. * From 6d80a3158c20318032fc282404a716fe418755ef Mon Sep 17 00:00:00 2001 From: Toan Le Date: Sun, 30 Aug 2026 09:56:17 +0700 Subject: [PATCH 2/2] Exclude vendored JSON.java from apache-rat to unblock trunk precommit. JSON.java was added to hadoop-hdfs-rbf by HADOOP-19951 (#8654) carrying its upstream Jetty EPL/Apache dual-license header, which apache-rat does not recognise as an ASF header. The asflicense check therefore reports one warning on every pull request targeting trunk, not just this one. Exclude the file via apache-rat-plugin the same way other vendored sources are excluded elsewhere in the project. This commit is separable from the HADOOP-19973 fix and can be dropped if it is preferred to address the breakage under its own JIRA. --- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 314fb127c651ee..e24908cba9d6b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -351,6 +351,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> + + org.apache.rat + apache-rat-plugin + + + src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/JSON.java + + +