HDFS-17957. Shut down leaked mini-cluster instances and review timeout values in hadoop-hdfs-project tests.

hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java

@@ -41,34 +41,33 @@ public class TestMountd {
   public void testStart() throws IOException {
     // Start minicluster
     NfsConfiguration config = new NfsConfiguration();
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(config).numDataNodes(1)
-        .build();
-    cluster.waitActive();
-
-    // Use emphral port in case tests are running in parallel
-    config.setInt("nfs3.mountd.port", 0);
-    config.setInt("nfs3.server.port", 0);
-
-    int newTimeoutMillis = 1000; // 1s
-    // Set the new portmap rpc timeout values and check
-    config.setInt(NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
-                  newTimeoutMillis);
-    assertTrue(config.getInt(
-                      NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
-          0) == newTimeoutMillis);
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
+        .numDataNodes(1).build()) {
+      cluster.waitActive();
 
-    // Start nfs
-    Nfs3 nfs3 = new Nfs3(config);
-    nfs3.startServiceInternal(false);
+      // Use emphral port in case tests are running in parallel
+      config.setInt("nfs3.mountd.port", 0);
+      config.setInt("nfs3.server.port", 0);
 
-    RpcProgramMountd mountd = (RpcProgramMountd) nfs3.getMountd()
-        .getRpcProgram();
-    mountd.nullOp(new XDR(), 1234, InetAddress.getByName("localhost"));
-    assertTrue(mountd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
-    RpcProgramNfs3 nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
-    nfsd.nullProcedure();
-    assertTrue(nfsd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
-
-    cluster.shutdown();
+      int newTimeoutMillis = 1000; // 1s
+      // Set the new portmap rpc timeout values and check
+      config.setInt(NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
+                    newTimeoutMillis);
+      assertTrue(config.getInt(
+                        NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
+            0) == newTimeoutMillis);
+
+      // Start nfs
+      Nfs3 nfs3 = new Nfs3(config);
+      nfs3.startServiceInternal(false);
+
+      RpcProgramMountd mountd = (RpcProgramMountd) nfs3.getMountd()
+          .getRpcProgram();
+      mountd.nullOp(new XDR(), 1234, InetAddress.getByName("localhost"));
+      assertTrue(mountd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
+      RpcProgramNfs3 nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
+      nfsd.nullProcedure();
+      assertTrue(nfsd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
+    }
   }
 }

hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml

@@ -203,6 +203,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <configuration>
           <systemPropertyVariables>
             <derby.stream.error.file>${project.build.directory}/derby.log</derby.stream.error.file>
+            <!-- Fail hung tests as named, per-method timeouts well before
+                 Surefire's forkedProcessTimeoutInSeconds kills the fork
+                 and loses the class's results.  See hadoop-hdfs/pom.xml.
+                 1800s rather than hadoop-hdfs's 600s: this module's fork
+                 budget is 3600s precisely because RBF methods and
+                 lifecycle setups legitimately run long, and a default
+                 below their honest runtime would convert slow tests into
+                 new named flakes. -->
+            <junit.jupiter.execution.timeout.default>1800 s</junit.jupiter.execution.timeout.default>
           </systemPropertyVariables>
         </configuration>
       </plugin>

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java

@@ -102,7 +102,12 @@ public static void globalSetUp() throws Exception {
 
   @AfterAll
   public static void tearDown() {
-    cluster.stopRouter(routerContext);
+    try {
+      cluster.stopRouter(routerContext);
+    } finally {
+      cluster.shutdown();
+      cluster = null;
+    }
   }
 
   @BeforeEach

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java

@@ -166,7 +166,12 @@ private static void setUpMocks()
 
   @AfterAll
   public static void tearDown() {
-    cluster.stopRouter(routerContext);
+    try {
+      cluster.stopRouter(routerContext);
+    } finally {
+      cluster.shutdown();
+      cluster = null;
+    }
   }
 
   @BeforeEach

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java

@@ -104,17 +104,21 @@ public static void tearDown() throws IOException {
   public void testNamenodeHeartbeatService() throws IOException {
 
     MiniRouterDFSCluster testCluster = new MiniRouterDFSCluster(true, 1);
-    Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
-        NAMESERVICES[0]);
-    NamenodeHeartbeatService server = new NamenodeHeartbeatService(
-        namenodeResolver, NAMESERVICES[0], NAMENODES[0]);
-    server.init(heartbeatConfig);
-    assertEquals(STATE.INITED, server.getServiceState());
-    server.start();
-    assertEquals(STATE.STARTED, server.getServiceState());
-    server.stop();
-    assertEquals(STATE.STOPPED, server.getServiceState());
-    server.close();
+    try {
+      Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
+          NAMESERVICES[0]);
+      NamenodeHeartbeatService server = new NamenodeHeartbeatService(
+          namenodeResolver, NAMESERVICES[0], NAMENODES[0]);
+      server.init(heartbeatConfig);
+      assertEquals(STATE.INITED, server.getServiceState());
+      server.start();
+      assertEquals(STATE.STARTED, server.getServiceState());
+      server.stop();
+      assertEquals(STATE.STOPPED, server.getServiceState());
+      server.close();
+    } finally {
+      testCluster.shutdown();
+    }
   }
 
   @Test

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java

@@ -56,7 +56,7 @@
 /**
  * Test retry behavior of the Router RPC Client.
  */
-@Timeout(100000)
+@Timeout(100)
 public class TestRouterRPCClientRetries {
 
   private static StateStoreDFSCluster cluster;

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java

@@ -24,6 +24,7 @@
 import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
 import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
 import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
@@ -58,6 +59,14 @@ private void setupCluster(boolean ha)
     cluster.waitClusterUp();
   }
 
+  @AfterEach
+  public void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
   @Test
   public void testGetFileInfoWhenNsFailover() throws Exception {
     setupCluster(true);

hadoop-hdfs-project/hadoop-hdfs/pom.xml

@@ -266,6 +266,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <configuration>
           <systemPropertyVariables>
             <runningWithNative>${runningWithNative}</runningWithNative>
+            <!-- Fail hung tests as named, per-method timeouts well before
+                 Surefire's forkedProcessTimeoutInSeconds kills the fork
+                 and loses the class's results.  The default SAME_THREAD
+                 mode interrupts the test thread at the deadline, which
+                 aborts any interruptible hang; an uninterruptible hang
+                 (socket accept, native call) still falls through to the
+                 fork timeout.  An explicit @Timeout overrides the
+                 default. -->
+            <junit.jupiter.execution.timeout.default>600 s</junit.jupiter.execution.timeout.default>
           </systemPropertyVariables>
           <properties>
             <property>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java

@@ -1126,11 +1126,15 @@ protected void compute() {
   }
 
   /**
-   * Return the size of fork pool used for adding replica in map.
+   * Return the configured parallelism of the fork pool used for adding
+   * replica in map.  Deliberately not {@link ForkJoinPool#getPoolSize()}:
+   * that reports the worker threads currently started, which the pool
+   * grows lazily and shrinks again when idle, so asserting on it races
+   * against the pool's own thread management.
    */
   @VisibleForTesting
   public static int getAddReplicaForkPoolSize() {
-    return addReplicaThreadPool.getPoolSize();
+    return addReplicaThreadPool.getParallelism();
   }
 
   @VisibleForTesting

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java

@@ -40,30 +40,31 @@ public void testSymlinkHdfsDisable() throws Exception {
     conf.setBoolean(
         CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, false);
     // spin up minicluster, get dfs and filecontext
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
-    DistributedFileSystem dfs = cluster.getFileSystem();
-    FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf);
-    // Create test files/links
-    FileContextTestHelper helper = new FileContextTestHelper(
-        "/tmp/TestSymlinkHdfsDisable");
-    Path root = helper.getTestRootPath(fc);
-    Path target = new Path(root, "target");
-    Path link = new Path(root, "link");
-    DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD);
-    fc.createSymlink(target, link, false);
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+      DistributedFileSystem dfs = cluster.getFileSystem();
+      FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf);
+      // Create test files/links
+      FileContextTestHelper helper = new FileContextTestHelper(
+          "/tmp/TestSymlinkHdfsDisable");
+      Path root = helper.getTestRootPath(fc);
+      Path target = new Path(root, "target");
+      Path link = new Path(root, "link");
+      DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD);
+      fc.createSymlink(target, link, false);
 
-    // Try to resolve links with FileSystem and FileContext
-    try {
-      fc.open(link);
-      fail("Expected error when attempting to resolve link");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("resolution is disabled", e);
-    }
-    try {
-      dfs.open(link);
-      fail("Expected error when attempting to resolve link");
-    } catch (IOException e) {
-      GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+      // Try to resolve links with FileSystem and FileContext
+      try {
+        fc.open(link);
+        fail("Expected error when attempting to resolve link");
+      } catch (IOException e) {
+        GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+      }
+      try {
+        dfs.open(link);
+        fail("Expected error when attempting to resolve link");
+      } catch (IOException e) {
+        GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+      }
     }
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java

@@ -220,8 +220,15 @@ public void testGetQuotaUsageWithQuotaDefined() throws IOException {
 
   @AfterAll
   public static void cleanup() throws IOException {
-    fHdfs.delete(new Path(testFileName), true);
-    fHdfs.delete(notInMountpointPath, true);
+    try {
+      fHdfs.delete(new Path(testFileName), true);
+      fHdfs.delete(notInMountpointPath, true);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+        cluster = null;
+      }
+    }
   }
 
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java

@@ -118,9 +118,16 @@ public void testGetFileChecksum() throws IOException, URISyntaxException {
 
   @AfterAll
   public static void cleanup() throws IOException {
-    fHdfs.delete(new Path(testfilename), true);
-    fHdfs.delete(new Path(someFile), true);
-    fHdfs.delete(new Path(someFile + "other"), true);
+    try {
+      fHdfs.delete(new Path(testfilename), true);
+      fHdfs.delete(new Path(someFile), true);
+      fHdfs.delete(new Path(someFile + "other"), true);
+    } finally {
+      if (cluster != null) {
+        cluster.shutdown();
+        cluster = null;
+      }
+    }
   }
 
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java

@@ -26,10 +26,11 @@ public class ListingBenchmark {
 
   public static void main(String[] args) throws IOException {
     HdfsConfiguration conf = new HdfsConfiguration();
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(0)
         .format(true)
-        .build();
-    NameNode nn = cluster.getNameNode();
+        .build()) {
+      NameNode nn = cluster.getNameNode();
+    }
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java

@@ -73,9 +73,12 @@ public void setup() throws IOException {
     conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
     conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
     final int numDNs = dataBlocks + parityBlocks;
+    // Ephemeral ports: the fixed-port variant simpleHATopology(2, 50070)
+    // binds 50070-50073 and fails with BindException whenever anything
+    // else on the CI agent holds one of them.
     cluster = new MiniDFSCluster.Builder(conf)
         .numDataNodes(numDNs)
-        .nnTopology(MiniDFSNNTopology.simpleHATopology(2, 50070))
+        .nnTopology(MiniDFSNNTopology.simpleHATopology())
         .build();
     cluster.waitActive();
     cluster.transitionToActive(0);
@@ -156,7 +159,7 @@ public Boolean get() {
             return false;
           }
         }
-      }, 5000, 24000);
+      }, 5000, 60000);
     } catch (TimeoutException e) {
       throw new IOException("Timeout waiting for recoverLease()");
     }