diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java
index 4411a4ee7ec1c5..f663565f5dc130 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java
@@ -41,34 +41,33 @@ public class TestMountd {
public void testStart() throws IOException {
// Start minicluster
NfsConfiguration config = new NfsConfiguration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(config).numDataNodes(1)
- .build();
- cluster.waitActive();
-
- // Use emphral port in case tests are running in parallel
- config.setInt("nfs3.mountd.port", 0);
- config.setInt("nfs3.server.port", 0);
-
- int newTimeoutMillis = 1000; // 1s
- // Set the new portmap rpc timeout values and check
- config.setInt(NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
- newTimeoutMillis);
- assertTrue(config.getInt(
- NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
- 0) == newTimeoutMillis);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
+ .numDataNodes(1).build()) {
+ cluster.waitActive();
- // Start nfs
- Nfs3 nfs3 = new Nfs3(config);
- nfs3.startServiceInternal(false);
+ // Use emphral port in case tests are running in parallel
+ config.setInt("nfs3.mountd.port", 0);
+ config.setInt("nfs3.server.port", 0);
- RpcProgramMountd mountd = (RpcProgramMountd) nfs3.getMountd()
- .getRpcProgram();
- mountd.nullOp(new XDR(), 1234, InetAddress.getByName("localhost"));
- assertTrue(mountd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
- RpcProgramNfs3 nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
- nfsd.nullProcedure();
- assertTrue(nfsd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
-
- cluster.shutdown();
+ int newTimeoutMillis = 1000; // 1s
+ // Set the new portmap rpc timeout values and check
+ config.setInt(NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
+ newTimeoutMillis);
+ assertTrue(config.getInt(
+ NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY,
+ 0) == newTimeoutMillis);
+
+ // Start nfs
+ Nfs3 nfs3 = new Nfs3(config);
+ nfs3.startServiceInternal(false);
+
+ RpcProgramMountd mountd = (RpcProgramMountd) nfs3.getMountd()
+ .getRpcProgram();
+ mountd.nullOp(new XDR(), 1234, InetAddress.getByName("localhost"));
+ assertTrue(mountd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
+ RpcProgramNfs3 nfsd = (RpcProgramNfs3) nfs3.getRpcProgram();
+ nfsd.nullProcedure();
+ assertTrue(nfsd.getPortmapUdpTimeoutMillis() == newTimeoutMillis);
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 556c4f3d175925..61f57e1aef4476 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -203,6 +203,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
${project.build.directory}/derby.log
+
+ 1800 s
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java
index b305612d69e1e8..1ccdac6efcb22b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java
@@ -102,7 +102,12 @@ public static void globalSetUp() throws Exception {
@AfterAll
public static void tearDown() {
- cluster.stopRouter(routerContext);
+ try {
+ cluster.stopRouter(routerContext);
+ } finally {
+ cluster.shutdown();
+ cluster = null;
+ }
}
@BeforeEach
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
index 8a4b06cc2f2763..0a7605a022a6dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
@@ -166,7 +166,12 @@ private static void setUpMocks()
@AfterAll
public static void tearDown() {
- cluster.stopRouter(routerContext);
+ try {
+ cluster.stopRouter(routerContext);
+ } finally {
+ cluster.shutdown();
+ cluster = null;
+ }
}
@BeforeEach
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
index 53770855236d24..51e6a6ac2b593a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
@@ -104,17 +104,21 @@ public static void tearDown() throws IOException {
public void testNamenodeHeartbeatService() throws IOException {
MiniRouterDFSCluster testCluster = new MiniRouterDFSCluster(true, 1);
- Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
- NAMESERVICES[0]);
- NamenodeHeartbeatService server = new NamenodeHeartbeatService(
- namenodeResolver, NAMESERVICES[0], NAMENODES[0]);
- server.init(heartbeatConfig);
- assertEquals(STATE.INITED, server.getServiceState());
- server.start();
- assertEquals(STATE.STARTED, server.getServiceState());
- server.stop();
- assertEquals(STATE.STOPPED, server.getServiceState());
- server.close();
+ try {
+ Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration(
+ NAMESERVICES[0]);
+ NamenodeHeartbeatService server = new NamenodeHeartbeatService(
+ namenodeResolver, NAMESERVICES[0], NAMENODES[0]);
+ server.init(heartbeatConfig);
+ assertEquals(STATE.INITED, server.getServiceState());
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ server.close();
+ } finally {
+ testCluster.shutdown();
+ }
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
index bdfb8165404a53..360fa3227bc070 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java
@@ -56,7 +56,7 @@
/**
* Test retry behavior of the Router RPC Client.
*/
-@Timeout(100000)
+@Timeout(100)
public class TestRouterRPCClientRetries {
private static StateStoreDFSCluster cluster;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java
index 64714c65d827f0..75325596f35ea8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
@@ -58,6 +59,14 @@ private void setupCluster(boolean ha)
cluster.waitClusterUp();
}
+ @AfterEach
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
@Test
public void testGetFileInfoWhenNsFailover() throws Exception {
setupCluster(true);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index e5a3cfa2d5fe91..64f8c6c6092a00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -266,6 +266,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
${runningWithNative}
+
+ 600 s
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index 8c643e9e16acef..fe4bd3e121dd39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -1126,11 +1126,15 @@ protected void compute() {
}
/**
- * Return the size of fork pool used for adding replica in map.
+ * Return the configured parallelism of the fork pool used for adding
+ * replica in map. Deliberately not {@link ForkJoinPool#getPoolSize()}:
+ * that reports the worker threads currently started, which the pool
+ * grows lazily and shrinks again when idle, so asserting on it races
+ * against the pool's own thread management.
*/
@VisibleForTesting
public static int getAddReplicaForkPoolSize() {
- return addReplicaThreadPool.getPoolSize();
+ return addReplicaThreadPool.getParallelism();
}
@VisibleForTesting
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java
index bfdfb688aca8e7..ac34f4cbf9b211 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java
@@ -40,30 +40,31 @@ public void testSymlinkHdfsDisable() throws Exception {
conf.setBoolean(
CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, false);
// spin up minicluster, get dfs and filecontext
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- DistributedFileSystem dfs = cluster.getFileSystem();
- FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf);
- // Create test files/links
- FileContextTestHelper helper = new FileContextTestHelper(
- "/tmp/TestSymlinkHdfsDisable");
- Path root = helper.getTestRootPath(fc);
- Path target = new Path(root, "target");
- Path link = new Path(root, "link");
- DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD);
- fc.createSymlink(target, link, false);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf);
+ // Create test files/links
+ FileContextTestHelper helper = new FileContextTestHelper(
+ "/tmp/TestSymlinkHdfsDisable");
+ Path root = helper.getTestRootPath(fc);
+ Path target = new Path(root, "target");
+ Path link = new Path(root, "link");
+ DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD);
+ fc.createSymlink(target, link, false);
- // Try to resolve links with FileSystem and FileContext
- try {
- fc.open(link);
- fail("Expected error when attempting to resolve link");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("resolution is disabled", e);
- }
- try {
- dfs.open(link);
- fail("Expected error when attempting to resolve link");
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+ // Try to resolve links with FileSystem and FileContext
+ try {
+ fc.open(link);
+ fail("Expected error when attempting to resolve link");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+ }
+ try {
+ dfs.open(link);
+ fail("Expected error when attempting to resolve link");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+ }
}
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
index e1881eea7e0f62..4dc7cc3edb7cc0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java
@@ -220,8 +220,15 @@ public void testGetQuotaUsageWithQuotaDefined() throws IOException {
@AfterAll
public static void cleanup() throws IOException {
- fHdfs.delete(new Path(testFileName), true);
- fHdfs.delete(notInMountpointPath, true);
+ try {
+ fHdfs.delete(new Path(testFileName), true);
+ fHdfs.delete(notInMountpointPath, true);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java
index 3bdeb0c47b1ef4..8bd759e977bf33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java
@@ -118,9 +118,16 @@ public void testGetFileChecksum() throws IOException, URISyntaxException {
@AfterAll
public static void cleanup() throws IOException {
- fHdfs.delete(new Path(testfilename), true);
- fHdfs.delete(new Path(someFile), true);
- fHdfs.delete(new Path(someFile + "other"), true);
+ try {
+ fHdfs.delete(new Path(testfilename), true);
+ fHdfs.delete(new Path(someFile), true);
+ fHdfs.delete(new Path(someFile + "other"), true);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java
index 67deaf3c3ed423..16711423313e62 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java
@@ -26,10 +26,11 @@ public class ListingBenchmark {
public static void main(String[] args) throws IOException {
HdfsConfiguration conf = new HdfsConfiguration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
.format(true)
- .build();
- NameNode nn = cluster.getNameNode();
+ .build()) {
+ NameNode nn = cluster.getNameNode();
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java
index 56fa0d49a01627..158ac0ee219afd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java
@@ -73,9 +73,12 @@ public void setup() throws IOException {
conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
final int numDNs = dataBlocks + parityBlocks;
+ // Ephemeral ports: the fixed-port variant simpleHATopology(2, 50070)
+ // binds 50070-50073 and fails with BindException whenever anything
+ // else on the CI agent holds one of them.
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(numDNs)
- .nnTopology(MiniDFSNNTopology.simpleHATopology(2, 50070))
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
.build();
cluster.waitActive();
cluster.transitionToActive(0);
@@ -156,7 +159,7 @@ public Boolean get() {
return false;
}
}
- }, 5000, 24000);
+ }, 5000, 60000);
} catch (TimeoutException e) {
throw new IOException("Timeout waiting for recoverLease()");
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java
index cfd3c53c075756..612fe09392251a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java
@@ -49,67 +49,66 @@ public void testDFSAddressConfig() throws IOException {
/*-------------------------------------------------------------------------
* By default, the DataNode socket address should be localhost (127.0.0.1).
*------------------------------------------------------------------------*/
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- cluster.waitActive();
-
- ArrayList dns = cluster.getDataNodes();
- DataNode dn = dns.get(0);
-
- String selfSocketAddr = dn.getXferAddress().toString();
- System.out.println("DN Self Socket Addr == " + selfSocketAddr);
- assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
-
- /*-------------------------------------------------------------------------
- * Shut down the datanodes, reconfigure, and bring them back up.
- * Even if told to use the configuration properties for dfs.datanode,
- * MiniDFSCluster.startDataNodes() should use localhost as the default if
- * the dfs.datanode properties are not set.
- *------------------------------------------------------------------------*/
- for (int i = 0; i < dns.size(); i++) {
- DataNodeProperties dnp = cluster.stopDataNode(i);
- assertNotNull(dnp, "Should have been able to stop simulated datanode");
- }
-
- conf.unset(DFS_DATANODE_ADDRESS_KEY);
- conf.unset(DFS_DATANODE_HTTP_ADDRESS_KEY);
- conf.unset(DFS_DATANODE_IPC_ADDRESS_KEY);
-
- cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR,
- null, null, null, false, true);
-
- dns = cluster.getDataNodes();
- dn = dns.get(0);
-
- selfSocketAddr = dn.getXferAddress().toString();
- System.out.println("DN Self Socket Addr == " + selfSocketAddr);
- // assert that default self socket address is 127.0.0.1
- assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
-
- /*-------------------------------------------------------------------------
- * Shut down the datanodes, reconfigure, and bring them back up.
- * This time, modify the dfs.datanode properties and make sure that they
- * are used to configure sockets by MiniDFSCluster.startDataNodes().
- *------------------------------------------------------------------------*/
- for (int i = 0; i < dns.size(); i++) {
- DataNodeProperties dnp = cluster.stopDataNode(i);
- assertNotNull(dnp, "Should have been able to stop simulated datanode");
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+ cluster.waitActive();
+
+ ArrayList dns = cluster.getDataNodes();
+ DataNode dn = dns.get(0);
+
+ String selfSocketAddr = dn.getXferAddress().toString();
+ System.out.println("DN Self Socket Addr == " + selfSocketAddr);
+ assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
+
+ /*-----------------------------------------------------------------------
+ * Shut down the datanodes, reconfigure, and bring them back up.
+ * Even if told to use the configuration properties for dfs.datanode,
+ * MiniDFSCluster.startDataNodes() should use localhost as the default if
+ * the dfs.datanode properties are not set.
+ *----------------------------------------------------------------------*/
+ for (int i = 0; i < dns.size(); i++) {
+ DataNodeProperties dnp = cluster.stopDataNode(i);
+ assertNotNull(dnp, "Should have been able to stop simulated datanode");
+ }
+
+ conf.unset(DFS_DATANODE_ADDRESS_KEY);
+ conf.unset(DFS_DATANODE_HTTP_ADDRESS_KEY);
+ conf.unset(DFS_DATANODE_IPC_ADDRESS_KEY);
+
+ cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR,
+ null, null, null, false, true);
+
+ dns = cluster.getDataNodes();
+ dn = dns.get(0);
+
+ selfSocketAddr = dn.getXferAddress().toString();
+ System.out.println("DN Self Socket Addr == " + selfSocketAddr);
+ // assert that default self socket address is 127.0.0.1
+ assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
+
+ /*-----------------------------------------------------------------------
+ * Shut down the datanodes, reconfigure, and bring them back up.
+ * This time, modify the dfs.datanode properties and make sure that they
+ * are used to configure sockets by MiniDFSCluster.startDataNodes().
+ *----------------------------------------------------------------------*/
+ for (int i = 0; i < dns.size(); i++) {
+ DataNodeProperties dnp = cluster.stopDataNode(i);
+ assertNotNull(dnp, "Should have been able to stop simulated datanode");
+ }
+
+ conf.set(DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0");
+ conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0");
+
+ cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR,
+ null, null, null, false, true);
+
+ dns = cluster.getDataNodes();
+ dn = dns.get(0);
+
+ selfSocketAddr = dn.getXferAddress().toString();
+ System.out.println("DN Self Socket Addr == " + selfSocketAddr);
+ // assert that default self socket address is 0.0.0.0
+ assertTrue(selfSocketAddr.contains("/0.0.0.0:"));
}
-
- conf.set(DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0");
- conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
- conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0");
-
- cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR,
- null, null, null, false, true);
-
- dns = cluster.getDataNodes();
- dn = dns.get(0);
-
- selfSocketAddr = dn.getXferAddress().toString();
- System.out.println("DN Self Socket Addr == " + selfSocketAddr);
- // assert that default self socket address is 0.0.0.0
- assertTrue(selfSocketAddr.contains("/0.0.0.0:"));
-
- cluster.shutdown();
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
index 5453a7465e86d3..51683607a25946 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
@@ -656,86 +656,89 @@ public void testPreserveEditLogs() throws Exception {
*/
Configuration conf = new HdfsConfiguration();
conf = UpgradeUtilities.initializeStorageStateConf(1, conf);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0)
.format(false)
.manageDataDfsDirs(false)
.manageNameDfsDirs(false)
.startupOption(StartupOption.UPGRADE)
- .build();
- DFSInotifyEventInputStream ieis =
- cluster.getFileSystem().getInotifyEventStream(0);
-
- EventBatch batch;
- Event.CreateEvent ce;
- Event.RenameEvent re;
+ .build()) {
+ DFSInotifyEventInputStream ieis =
+ cluster.getFileSystem().getInotifyEventStream(0);
- // mkdir /input
- batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
- assertEquals(1, batch.getEvents().length);
- assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
- ce = (Event.CreateEvent) batch.getEvents()[0];
- assertEquals(ce.getPath(), "/input");
+ EventBatch batch;
+ Event.CreateEvent ce;
+ Event.RenameEvent re;
- // mkdir /input/dir1~5
- for (int i = 1; i <= 5; i++) {
+ // mkdir /input
batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
assertEquals(1, batch.getEvents().length);
assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
ce = (Event.CreateEvent) batch.getEvents()[0];
- assertEquals(ce.getPath(), "/input/dir" + i);
- }
- // copyFromLocal randome_file_1~2 /input/dir1~2
- for (int i = 1; i <= 2; i++) {
- batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
- assertEquals(1, batch.getEvents().length);
- if (batch.getEvents()[0].getEventType() != Event.EventType.CREATE) {
- FSImage.LOG.debug("");
+ assertEquals(ce.getPath(), "/input");
+
+ // mkdir /input/dir1~5
+ for (int i = 1; i <= 5; i++) {
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(
+ batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ ce = (Event.CreateEvent) batch.getEvents()[0];
+ assertEquals(ce.getPath(), "/input/dir" + i);
}
- assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ // copyFromLocal randome_file_1~2 /input/dir1~2
+ for (int i = 1; i <= 2; i++) {
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ if (batch.getEvents()[0].getEventType() != Event.EventType.CREATE) {
+ FSImage.LOG.debug("");
+ }
+ assertTrue(
+ batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
- // copyFromLocal randome_file_1 /input/dir1, CLOSE
- batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
- assertEquals(1, batch.getEvents().length);
- assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CLOSE);
+ // copyFromLocal randome_file_1 /input/dir1, CLOSE
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(
+ batch.getEvents()[0].getEventType() == Event.EventType.CLOSE);
- // copyFromLocal randome_file_1 /input/dir1, CLOSE
+ // copyFromLocal randome_file_1 /input/dir1, CLOSE
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() ==
+ Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir" + i + "/randome_file_" + i);
+ }
+
+ // mv /input/dir1/randome_file_1 /input/dir3/randome_file_3
+ long txIDBeforeRename = batch.getTxid();
batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
assertEquals(1, batch.getEvents().length);
- assertTrue(batch.getEvents()[0].getEventType() ==
- Event.EventType.RENAME);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
re = (Event.RenameEvent) batch.getEvents()[0];
- assertEquals(re.getDstPath(), "/input/dir" + i + "/randome_file_" + i);
- }
-
- // mv /input/dir1/randome_file_1 /input/dir3/randome_file_3
- long txIDBeforeRename = batch.getTxid();
- batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
- assertEquals(1, batch.getEvents().length);
- assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
- re = (Event.RenameEvent) batch.getEvents()[0];
- assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
+ assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
- // rmdir /input/dir1
- batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
- assertEquals(1, batch.getEvents().length);
- assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.UNLINK);
- assertEquals(((Event.UnlinkEvent) batch.getEvents()[0]).getPath(),
- "/input/dir1");
- long lastTxID = batch.getTxid();
-
- // Start inotify from the tx before rename /input/dir1/randome_file_1
- ieis = cluster.getFileSystem().getInotifyEventStream(txIDBeforeRename);
- batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
- assertEquals(1, batch.getEvents().length);
- assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
- re = (Event.RenameEvent) batch.getEvents()[0];
- assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
+ // rmdir /input/dir1
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.UNLINK);
+ assertEquals(((Event.UnlinkEvent) batch.getEvents()[0]).getPath(),
+ "/input/dir1");
+ long lastTxID = batch.getTxid();
- // Try to read beyond available edits
- ieis = cluster.getFileSystem().getInotifyEventStream(lastTxID + 1);
- assertNull(ieis.poll());
+ // Start inotify from the tx before rename /input/dir1/randome_file_1
+ ieis = cluster.getFileSystem().getInotifyEventStream(txIDBeforeRename);
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
- cluster.shutdown();
+ // Try to read beyond available edits
+ ieis = cluster.getFileSystem().getInotifyEventStream(lastTxID + 1);
+ assertNull(ieis.poll());
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 91bc36042e22ef..089c1cffb2dd4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1260,133 +1260,136 @@ public void testFileChecksum() throws Exception {
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(2).build();
- final FileSystem hdfs = cluster.getFileSystem();
-
- final String nnAddr = conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
- final UserGroupInformation current = UserGroupInformation.getCurrentUser();
- final UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
- current.getShortUserName() + "x", new String[]{"user"});
-
try {
- hdfs.getFileChecksum(new Path(
- "/test/TestNonExistingFile"));
- fail("Expecting FileNotFoundException");
- } catch (FileNotFoundException e) {
- assertTrue(e.getMessage().contains("File does not exist: /test/TestNonExistingFile"),
- "Not throwing the intended exception message");
- }
+ final FileSystem hdfs = cluster.getFileSystem();
- try {
- Path path = new Path("/test/TestExistingDir/");
- hdfs.mkdirs(path);
- hdfs.getFileChecksum(path);
- fail("Expecting FileNotFoundException");
- } catch (FileNotFoundException e) {
- assertTrue(e.getMessage().contains("Path is not a file: /test/TestExistingDir"),
- "Not throwing the intended exception message");
- }
-
- //webhdfs
- final String webhdfsuri = WebHdfsConstants.WEBHDFS_SCHEME + "://" + nnAddr;
- System.out.println("webhdfsuri=" + webhdfsuri);
- final FileSystem webhdfs = ugi.doAs(
- new PrivilegedExceptionAction() {
- @Override
- public FileSystem run() throws Exception {
- return new Path(webhdfsuri).getFileSystem(conf);
+ final String nnAddr = conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
+ final UserGroupInformation current = UserGroupInformation.getCurrentUser();
+ final UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+ current.getShortUserName() + "x", new String[]{"user"});
+
+ try {
+ hdfs.getFileChecksum(new Path(
+ "/test/TestNonExistingFile"));
+ fail("Expecting FileNotFoundException");
+ } catch (FileNotFoundException e) {
+ assertTrue(e.getMessage().contains("File does not exist: /test/TestNonExistingFile"),
+ "Not throwing the intended exception message");
}
- });
-
- final Path dir = new Path("/filechecksum");
- final int block_size = 1024;
- final int buffer_size = conf.getInt(
- CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096);
- conf.setInt(HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 512);
-
- //try different number of blocks
- for(int n = 0; n < 5; n++) {
- //generate random data
- final byte[] data = new byte[RAN.nextInt(block_size/2-1)+n*block_size+1];
- RAN.nextBytes(data);
- System.out.println("data.length=" + data.length);
-
- //write data to a file
- final Path foo = new Path(dir, "foo" + n);
- {
- final FSDataOutputStream out = hdfs.create(foo, false, buffer_size,
- (short)2, block_size);
- out.write(data);
- out.close();
+
+ try {
+ Path path = new Path("/test/TestExistingDir/");
+ hdfs.mkdirs(path);
+ hdfs.getFileChecksum(path);
+ fail("Expecting FileNotFoundException");
+ } catch (FileNotFoundException e) {
+ assertTrue(e.getMessage().contains("Path is not a file: /test/TestExistingDir"),
+ "Not throwing the intended exception message");
}
-
- //compute checksum
- final FileChecksum hdfsfoocs = hdfs.getFileChecksum(foo);
- System.out.println("hdfsfoocs=" + hdfsfoocs);
//webhdfs
- final FileChecksum webhdfsfoocs = webhdfs.getFileChecksum(foo);
- System.out.println("webhdfsfoocs=" + webhdfsfoocs);
+ final String webhdfsuri = WebHdfsConstants.WEBHDFS_SCHEME + "://" + nnAddr;
+ System.out.println("webhdfsuri=" + webhdfsuri);
+ final FileSystem webhdfs = ugi.doAs(
+ new PrivilegedExceptionAction() {
+ @Override
+ public FileSystem run() throws Exception {
+ return new Path(webhdfsuri).getFileSystem(conf);
+ }
+ });
+
+ final Path dir = new Path("/filechecksum");
+ final int block_size = 1024;
+ final int buffer_size = conf.getInt(
+ CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096);
+ conf.setInt(HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 512);
+
+ //try different number of blocks
+ for(int n = 0; n < 5; n++) {
+ //generate random data
+ final byte[] data = new byte[RAN.nextInt(block_size/2-1)+n*block_size+1];
+ RAN.nextBytes(data);
+ System.out.println("data.length=" + data.length);
+
+ //write data to a file
+ final Path foo = new Path(dir, "foo" + n);
+ {
+ final FSDataOutputStream out = hdfs.create(foo, false, buffer_size,
+ (short)2, block_size);
+ out.write(data);
+ out.close();
+ }
- final Path webhdfsqualified = new Path(webhdfsuri + dir, "foo" + n);
- final FileChecksum webhdfs_qfoocs =
- webhdfs.getFileChecksum(webhdfsqualified);
- System.out.println("webhdfs_qfoocs=" + webhdfs_qfoocs);
+ //compute checksum
+ final FileChecksum hdfsfoocs = hdfs.getFileChecksum(foo);
+ System.out.println("hdfsfoocs=" + hdfsfoocs);
- //create a zero byte file
- final Path zeroByteFile = new Path(dir, "zeroByteFile" + n);
- {
- final FSDataOutputStream out = hdfs.create(zeroByteFile, false,
- buffer_size, (short)2, block_size);
- out.close();
- }
+ //webhdfs
+ final FileChecksum webhdfsfoocs = webhdfs.getFileChecksum(foo);
+ System.out.println("webhdfsfoocs=" + webhdfsfoocs);
- //write another file
- final Path bar = new Path(dir, "bar" + n);
- {
- final FSDataOutputStream out = hdfs.create(bar, false, buffer_size,
- (short)2, block_size);
- out.write(data);
- out.close();
- }
+ final Path webhdfsqualified = new Path(webhdfsuri + dir, "foo" + n);
+ final FileChecksum webhdfs_qfoocs =
+ webhdfs.getFileChecksum(webhdfsqualified);
+ System.out.println("webhdfs_qfoocs=" + webhdfs_qfoocs);
- {
- final FileChecksum zeroChecksum = hdfs.getFileChecksum(zeroByteFile);
- final String magicValue =
- "MD5-of-0MD5-of-0CRC32:70bc8f4b72a86921468bf8e8441dce51";
- // verify the magic val for zero byte files
- assertEquals(magicValue, zeroChecksum.toString());
-
- //verify checksums for empty file and 0 request length
- final FileChecksum checksumWith0 = hdfs.getFileChecksum(bar, 0);
- assertEquals(zeroChecksum, checksumWith0);
-
- //verify checksum
- final FileChecksum barcs = hdfs.getFileChecksum(bar);
- final int barhashcode = barcs.hashCode();
- assertEquals(hdfsfoocs.hashCode(), barhashcode);
- assertEquals(hdfsfoocs, barcs);
+ //create a zero byte file
+ final Path zeroByteFile = new Path(dir, "zeroByteFile" + n);
+ {
+ final FSDataOutputStream out = hdfs.create(zeroByteFile, false,
+ buffer_size, (short)2, block_size);
+ out.close();
+ }
- //webhdfs
- assertEquals(webhdfsfoocs.hashCode(), barhashcode);
- assertEquals(webhdfsfoocs, barcs);
+ //write another file
+ final Path bar = new Path(dir, "bar" + n);
+ {
+ final FSDataOutputStream out = hdfs.create(bar, false, buffer_size,
+ (short)2, block_size);
+ out.write(data);
+ out.close();
+ }
- assertEquals(webhdfs_qfoocs.hashCode(), barhashcode);
- assertEquals(webhdfs_qfoocs, barcs);
- }
+ {
+ final FileChecksum zeroChecksum = hdfs.getFileChecksum(zeroByteFile);
+ final String magicValue =
+ "MD5-of-0MD5-of-0CRC32:70bc8f4b72a86921468bf8e8441dce51";
+ // verify the magic val for zero byte files
+ assertEquals(magicValue, zeroChecksum.toString());
+
+ //verify checksums for empty file and 0 request length
+ final FileChecksum checksumWith0 = hdfs.getFileChecksum(bar, 0);
+ assertEquals(zeroChecksum, checksumWith0);
+
+ //verify checksum
+ final FileChecksum barcs = hdfs.getFileChecksum(bar);
+ final int barhashcode = barcs.hashCode();
+ assertEquals(hdfsfoocs.hashCode(), barhashcode);
+ assertEquals(hdfsfoocs, barcs);
+
+ //webhdfs
+ assertEquals(webhdfsfoocs.hashCode(), barhashcode);
+ assertEquals(webhdfsfoocs, barcs);
+
+ assertEquals(webhdfs_qfoocs.hashCode(), barhashcode);
+ assertEquals(webhdfs_qfoocs, barcs);
+ }
- hdfs.setPermission(dir, new FsPermission((short)0));
+ hdfs.setPermission(dir, new FsPermission((short)0));
- { //test permission error on webhdfs
- try {
- webhdfs.getFileChecksum(webhdfsqualified);
- fail();
- } catch(IOException ioe) {
- FileSystem.LOG.info("GOOD: getting an exception", ioe);
+ { //test permission error on webhdfs
+ try {
+ webhdfs.getFileChecksum(webhdfsqualified);
+ fail();
+ } catch(IOException ioe) {
+ FileSystem.LOG.info("GOOD: getting an exception", ioe);
+ }
}
+ hdfs.setPermission(dir, new FsPermission((short)0777));
}
- hdfs.setPermission(dir, new FsPermission((short)0777));
+ } finally {
+ cluster.shutdown();
}
- cluster.shutdown();
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
index 3d2f286f411c4d..f6da6597598259 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
@@ -489,58 +489,59 @@ public void testReadSkipStaleStorage() throws Exception {
final Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(1)
.storagesPerDatanode(storageNum)
- .build();
- cluster.waitActive();
+ .build()) {
+ cluster.waitActive();
- FileSystem fs = cluster.getFileSystem();
- DFSTestUtil.createFile(fs, path, false, 1024, fileLen,
- BLOCK_SIZE, repFactor, 0, true);
+ FileSystem fs = cluster.getFileSystem();
+ DFSTestUtil.createFile(fs, path, false, 1024, fileLen,
+ BLOCK_SIZE, repFactor, 0, true);
- // get datanode info
- ClientProtocol client = NameNodeProxies.createProxy(conf,
- cluster.getFileSystem(0).getUri(),
- ClientProtocol.class).getProxy();
- DatanodeInfo[] dataNodes = client.getDatanodeReport(DatanodeReportType.ALL);
+ // get datanode info
+ ClientProtocol client = NameNodeProxies.createProxy(conf,
+ cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
+ DatanodeInfo[] dataNodes = client.getDatanodeReport(DatanodeReportType.ALL);
+
+ // get storage info
+ BlockManager bm0 = cluster.getNamesystem(0).getBlockManager();
+ DatanodeStorageInfo[] storageInfos = bm0.getDatanodeManager()
+ .getDatanode(dataNodes[0].getDatanodeUuid()).getStorageInfos();
- // get storage info
- BlockManager bm0 = cluster.getNamesystem(0).getBlockManager();
- DatanodeStorageInfo[] storageInfos = bm0.getDatanodeManager()
- .getDatanode(dataNodes[0].getDatanodeUuid()).getStorageInfos();
+ InetSocketAddress addr = new InetSocketAddress("localhost",
+ cluster.getNameNodePort());
+ NamenodeProtocol namenode = NameNodeProxies.createProxy(conf,
+ DFSUtilClient.getNNUri(addr), NamenodeProtocol.class).getProxy();
- InetSocketAddress addr = new InetSocketAddress("localhost",
- cluster.getNameNodePort());
- NamenodeProtocol namenode = NameNodeProxies.createProxy(conf,
- DFSUtilClient.getNNUri(addr), NamenodeProtocol.class).getProxy();
-
- // check blocks count equals to blockNum
- BlockWithLocations[] blocks = namenode.getBlocks(
- dataNodes[0], fileLen*2, 0, 0, null).getBlocks();
- assertEquals(blockNum, blocks.length);
-
- // calculate the block count on storage[0]
- int count = 0;
- for (BlockWithLocations b : blocks) {
- for (String s : b.getStorageIDs()) {
- if (s.equals(storageInfos[0].getStorageID())) {
- count++;
+ // check blocks count equals to blockNum
+ BlockWithLocations[] blocks = namenode.getBlocks(
+ dataNodes[0], fileLen*2, 0, 0, null).getBlocks();
+ assertEquals(blockNum, blocks.length);
+
+ // calculate the block count on storage[0]
+ int count = 0;
+ for (BlockWithLocations b : blocks) {
+ for (String s : b.getStorageIDs()) {
+ if (s.equals(storageInfos[0].getStorageID())) {
+ count++;
+ }
}
}
- }
- // set storage[0] stale
- storageInfos[0].setBlockContentsStale(true);
- blocks = namenode.getBlocks(
- dataNodes[0], fileLen*2, 0, 0, null).getBlocks();
- assertEquals(blockNum - count, blocks.length);
-
- // set all storage stale
- bm0.getDatanodeManager().markAllDatanodesStaleAndSetKeyUpdateIfNeed();
- blocks = namenode.getBlocks(
- dataNodes[0], fileLen*2, 0, 0, null).getBlocks();
- assertEquals(0, blocks.length);
+ // set storage[0] stale
+ storageInfos[0].setBlockContentsStale(true);
+ blocks = namenode.getBlocks(
+ dataNodes[0], fileLen*2, 0, 0, null).getBlocks();
+ assertEquals(blockNum - count, blocks.length);
+
+ // set all storage stale
+ bm0.getDatanodeManager().markAllDatanodesStaleAndSetKeyUpdateIfNeed();
+ blocks = namenode.getBlocks(
+ dataNodes[0], fileLen*2, 0, 0, null).getBlocks();
+ assertEquals(0, blocks.length);
+ }
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java
index 0992f54b891901..4a0b5eda9f9919 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java
@@ -52,58 +52,59 @@ public void testHAAuxiliaryPort() throws Exception {
.addNN(new MiniDFSNNTopology.NNConf("nn1"))
.addNN(new MiniDFSNNTopology.NNConf("nn2")));
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(topology)
.numDataNodes(0)
- .build();
- cluster.transitionToActive(0);
- cluster.waitActive();
+ .build()) {
+ cluster.transitionToActive(0);
+ cluster.waitActive();
- NameNode nn0 = cluster.getNameNode(0);
- NameNode nn1 = cluster.getNameNode(1);
+ NameNode nn0 = cluster.getNameNode(0);
+ NameNode nn1 = cluster.getNameNode(1);
- // all the addresses below are valid nn0 addresses
- NameNodeRpcServer rpcServer0 = (NameNodeRpcServer)nn0.getRpcServer();
- InetSocketAddress server0RpcAddress = rpcServer0.getRpcAddress();
- Set auxAddrServer0 =
- rpcServer0.getAuxiliaryRpcAddresses();
- assertEquals(2, auxAddrServer0.size());
+ // all the addresses below are valid nn0 addresses
+ NameNodeRpcServer rpcServer0 = (NameNodeRpcServer)nn0.getRpcServer();
+ InetSocketAddress server0RpcAddress = rpcServer0.getRpcAddress();
+ Set auxAddrServer0 =
+ rpcServer0.getAuxiliaryRpcAddresses();
+ assertEquals(2, auxAddrServer0.size());
- // all the addresses below are valid nn1 addresses
- NameNodeRpcServer rpcServer1 = (NameNodeRpcServer)nn1.getRpcServer();
- InetSocketAddress server1RpcAddress = rpcServer1.getRpcAddress();
- Set auxAddrServer1 =
- rpcServer1.getAuxiliaryRpcAddresses();
- assertEquals(2, auxAddrServer1.size());
+ // all the addresses below are valid nn1 addresses
+ NameNodeRpcServer rpcServer1 = (NameNodeRpcServer)nn1.getRpcServer();
+ InetSocketAddress server1RpcAddress = rpcServer1.getRpcAddress();
+ Set auxAddrServer1 =
+ rpcServer1.getAuxiliaryRpcAddresses();
+ assertEquals(2, auxAddrServer1.size());
- // mkdir on nn0 uri 0
- URI nn0URI = new URI("hdfs://localhost:" +
- server0RpcAddress.getPort());
- try (DFSClient client0 = new DFSClient(nn0URI, conf)){
- client0.mkdirs("/test", null, true);
- // should be available on other ports also
- for (InetSocketAddress auxAddr : auxAddrServer0) {
- nn0URI = new URI("hdfs://localhost:" + auxAddr.getPort());
- try (DFSClient clientTmp = new DFSClient(nn0URI, conf)) {
- assertTrue(clientTmp.exists("/test"));
+ // mkdir on nn0 uri 0
+ URI nn0URI = new URI("hdfs://localhost:" +
+ server0RpcAddress.getPort());
+ try (DFSClient client0 = new DFSClient(nn0URI, conf)){
+ client0.mkdirs("/test", null, true);
+ // should be available on other ports also
+ for (InetSocketAddress auxAddr : auxAddrServer0) {
+ nn0URI = new URI("hdfs://localhost:" + auxAddr.getPort());
+ try (DFSClient clientTmp = new DFSClient(nn0URI, conf)) {
+ assertTrue(clientTmp.exists("/test"));
+ }
}
}
- }
- // now perform a failover
- cluster.shutdownNameNode(0);
- cluster.transitionToActive(1);
+ // now perform a failover
+ cluster.shutdownNameNode(0);
+ cluster.transitionToActive(1);
- // then try to read the file from the nn1
- URI nn1URI = new URI("hdfs://localhost:" +
- server1RpcAddress.getPort());
- try (DFSClient client1 = new DFSClient(nn1URI, conf)) {
- assertTrue(client1.exists("/test"));
- // should be available on other ports also
- for (InetSocketAddress auxAddr : auxAddrServer1) {
- nn1URI = new URI("hdfs://localhost:" + auxAddr.getPort());
- try (DFSClient clientTmp = new DFSClient(nn1URI, conf)) {
- assertTrue(client1.exists("/test"));
+ // then try to read the file from the nn1
+ URI nn1URI = new URI("hdfs://localhost:" +
+ server1RpcAddress.getPort());
+ try (DFSClient client1 = new DFSClient(nn1URI, conf)) {
+ assertTrue(client1.exists("/test"));
+ // should be available on other ports also
+ for (InetSocketAddress auxAddr : auxAddrServer1) {
+ nn1URI = new URI("hdfs://localhost:" + auxAddr.getPort());
+ try (DFSClient clientTmp = new DFSClient(nn1URI, conf)) {
+ assertTrue(client1.exists("/test"));
+ }
}
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index be932c2267ac33..e04ee2855b3d8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -225,11 +225,11 @@ public void testIsClusterUpAfterShutdown() throws Throwable {
try {
DistributedFileSystem dfs = cluster4.getFileSystem();
dfs.setSafeMode(SafeModeAction.ENTER);
- cluster4.shutdown();
} finally {
+ cluster4.shutdown();
while(cluster4.isClusterUp()){
Thread.sleep(1000);
- }
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
index 8f5ba9018dfa7b..70145552517a32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
@@ -96,12 +96,12 @@ public void testEOFWithBlockReaderLocal() throws Exception {
try {
final Configuration conf = testContext.newConfiguration();
conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_CACHE_READAHEAD, BLOCK_SIZE);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
- .format(true).build();
- testEOF(cluster, 1);
- testEOF(cluster, 14);
- testEOF(cluster, 10000);
- cluster.shutdown();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(1).format(true).build()) {
+ testEOF(cluster, 1);
+ testEOF(cluster, 14);
+ testEOF(cluster, 10000);
+ }
} finally {
testContext.close();
}
@@ -112,12 +112,12 @@ public void testEOFWithBlockReaderLocal() throws Exception {
public void testEOFWithRemoteBlockReader() throws Exception {
final Configuration conf = new Configuration();
conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_CACHE_READAHEAD, BLOCK_SIZE);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
- .format(true).build();
- testEOF(cluster, 1);
- testEOF(cluster, 14);
- testEOF(cluster, 10000);
- cluster.shutdown();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(1).format(true).build()) {
+ testEOF(cluster, 1);
+ testEOF(cluster, 14);
+ testEOF(cluster, 10000);
+ }
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index 4ff9f07419fd7f..dcec782f9e6031 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -228,46 +228,49 @@ private void testBadBlockReportOnTransfer(
int replicaCount = 0;
short replFactor = 1;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- cluster.waitActive();
- fs = cluster.getFileSystem();
- dfsClient = new DFSClient(new InetSocketAddress("localhost",
- cluster.getNameNodePort()), conf);
+ try {
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ dfsClient = new DFSClient(new InetSocketAddress("localhost",
+ cluster.getNameNodePort()), conf);
- // Create file with replication factor of 1
- Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1");
- DFSTestUtil.createFile(fs, file1, 1024, replFactor, 0);
- DFSTestUtil.waitReplication(fs, file1, replFactor);
+ // Create file with replication factor of 1
+ Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1");
+ DFSTestUtil.createFile(fs, file1, 1024, replFactor, 0);
+ DFSTestUtil.waitReplication(fs, file1, replFactor);
- // Corrupt the block belonging to the created file
- ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
+ // Corrupt the block belonging to the created file
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
- int blockFilesCorrupted =
- corruptBlockByDeletingBlockFile?
- cluster.corruptBlockOnDataNodesByDeletingBlockFile(block) :
- cluster.corruptBlockOnDataNodes(block);
+ int blockFilesCorrupted =
+ corruptBlockByDeletingBlockFile?
+ cluster.corruptBlockOnDataNodesByDeletingBlockFile(block) :
+ cluster.corruptBlockOnDataNodes(block);
- assertEquals(replFactor, blockFilesCorrupted, "Corrupted too few blocks");
+ assertEquals(replFactor, blockFilesCorrupted, "Corrupted too few blocks");
- // Increase replication factor, this should invoke transfer request
- // Receiving datanode fails on checksum and reports it to namenode
- replFactor = 2;
- fs.setReplication(file1, replFactor);
+ // Increase replication factor, this should invoke transfer request
+ // Receiving datanode fails on checksum and reports it to namenode
+ replFactor = 2;
+ fs.setReplication(file1, replFactor);
- // Now get block details and check if the block is corrupt
- blocks = dfsClient.getNamenode().
- getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
- while (blocks.get(0).isCorrupt() != true) {
- try {
- LOG.info("Waiting until block is marked as corrupt...");
- Thread.sleep(1000);
- } catch (InterruptedException ie) {
- }
+ // Now get block details and check if the block is corrupt
blocks = dfsClient.getNamenode().
getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
+ while (blocks.get(0).isCorrupt() != true) {
+ try {
+ LOG.info("Waiting until block is marked as corrupt...");
+ Thread.sleep(1000);
+ } catch (InterruptedException ie) {
+ }
+ blocks = dfsClient.getNamenode().
+ getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
+ }
+ replicaCount = blocks.get(0).getLocations().length;
+ assertTrue(replicaCount == 1);
+ } finally {
+ cluster.shutdown();
}
- replicaCount = blocks.get(0).getLocations().length;
- assertTrue(replicaCount == 1);
- cluster.shutdown();
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
index a02c88f9d11959..c194b848d48958 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
@@ -221,7 +221,7 @@ public void testRollingUpgradeWithQJM() throws Exception {
final Path baz = new Path("/baz");
final RollingUpgradeInfo info1;
- {
+ try {
final DistributedFileSystem dfs = cluster.getFileSystem();
dfs.mkdirs(foo);
@@ -235,6 +235,7 @@ public void testRollingUpgradeWithQJM() throws Exception {
assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
dfs.mkdirs(bar);
+ } finally {
cluster.shutdown();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java
index 810f7e1864d174..905b83c9eabd38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java
@@ -133,19 +133,22 @@ public void testFallbackFromShortCircuitToUnixDomainTraffic()
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
- cluster.waitActive();
- FileSystem dfs = FileSystem.get(cluster.getURI(0), clientConf);
- String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 8193;
- final int SEED = 0xFADED;
- DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
- byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE));
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
- assertTrue(Arrays.equals(contents, expected));
- cluster.shutdown();
- sockDir.close();
+ try {
+ cluster.waitActive();
+ FileSystem dfs = FileSystem.get(cluster.getURI(0), clientConf);
+ String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 8193;
+ final int SEED = 0xFADED;
+ DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+ byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE));
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ assertTrue(Arrays.equals(contents, expected));
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
/**
@@ -210,42 +213,45 @@ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
"testMultipleWaitersOnShortCircuitCache", sockDir);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- final DistributedFileSystem dfs = cluster.getFileSystem();
- final String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 4000;
- final int SEED = 0xFADED;
- final int NUM_THREADS = 10;
- DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
- Runnable readerRunnable = new Runnable() {
- @Override
- public void run() {
- try {
- byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE));
- assertFalse(creationIsBlocked.get());
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
- assertTrue(Arrays.equals(contents, expected));
- } catch (Throwable e) {
- LOG.error("readerRunnable error", e);
- testFailed.set(true);
+ try {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 4000;
+ final int SEED = 0xFADED;
+ final int NUM_THREADS = 10;
+ DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+ Runnable readerRunnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE));
+ assertFalse(creationIsBlocked.get());
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ assertTrue(Arrays.equals(contents, expected));
+ } catch (Throwable e) {
+ LOG.error("readerRunnable error", e);
+ testFailed.set(true);
+ }
}
+ };
+ Thread threads[] = new Thread[NUM_THREADS];
+ for (int i = 0; i < NUM_THREADS; i++) {
+ threads[i] = new Thread(readerRunnable);
+ threads[i].start();
}
- };
- Thread threads[] = new Thread[NUM_THREADS];
- for (int i = 0; i < NUM_THREADS; i++) {
- threads[i] = new Thread(readerRunnable);
- threads[i].start();
- }
- Thread.sleep(500);
- latch.countDown();
- for (int i = 0; i < NUM_THREADS; i++) {
- Uninterruptibles.joinUninterruptibly(threads[i]);
+ Thread.sleep(500);
+ latch.countDown();
+ for (int i = 0; i < NUM_THREADS; i++) {
+ Uninterruptibles.joinUninterruptibly(threads[i]);
+ }
+ assertFalse(testFailed.get());
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
}
- cluster.shutdown();
- sockDir.close();
- assertFalse(testFailed.get());
}
/**
@@ -281,71 +287,74 @@ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
"testShortCircuitCacheTemporaryFailure", sockDir);
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- final DistributedFileSystem dfs = cluster.getFileSystem();
- final String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 4000;
- final int NUM_THREADS = 2;
- final int SEED = 0xFADED;
- final CountDownLatch gotFailureLatch = new CountDownLatch(NUM_THREADS);
- final CountDownLatch shouldRetryLatch = new CountDownLatch(1);
- DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
- Runnable readerRunnable = new Runnable() {
- @Override
- public void run() {
- try {
- // First time should fail.
- List locatedBlocks =
- cluster.getNameNode().getRpcServer().getBlockLocations(
- TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks();
- LocatedBlock lblock = locatedBlocks.get(0); // first block
- BlockReader blockReader = null;
+ try {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 4000;
+ final int NUM_THREADS = 2;
+ final int SEED = 0xFADED;
+ final CountDownLatch gotFailureLatch = new CountDownLatch(NUM_THREADS);
+ final CountDownLatch shouldRetryLatch = new CountDownLatch(1);
+ DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+ Runnable readerRunnable = new Runnable() {
+ @Override
+ public void run() {
try {
- blockReader = BlockReaderTestUtil.getBlockReader(
- cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
- fail("expected getBlockReader to fail the first time.");
- } catch (Throwable t) {
- assertTrue(t.getMessage().contains("TCP reads were disabled for testing"),
- "expected to see 'TCP reads were disabled "
- + "for testing' in exception " + t);
- } finally {
- if (blockReader != null) blockReader.close(); // keep findbugs happy
- }
- gotFailureLatch.countDown();
- shouldRetryLatch.await();
+ // First time should fail.
+ List locatedBlocks =
+ cluster.getNameNode().getRpcServer().getBlockLocations(
+ TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks();
+ LocatedBlock lblock = locatedBlocks.get(0); // first block
+ BlockReader blockReader = null;
+ try {
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
+ fail("expected getBlockReader to fail the first time.");
+ } catch (Throwable t) {
+ assertTrue(t.getMessage().contains("TCP reads were disabled for testing"),
+ "expected to see 'TCP reads were disabled "
+ + "for testing' in exception " + t);
+ } finally {
+ if (blockReader != null) blockReader.close(); // keep findbugs happy
+ }
+ gotFailureLatch.countDown();
+ shouldRetryLatch.await();
- // Second time should succeed.
- try {
- blockReader = BlockReaderTestUtil.getBlockReader(
- cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
+ // Second time should succeed.
+ try {
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
+ } catch (Throwable t) {
+ LOG.error("error trying to retrieve a block reader " +
+ "the second time.", t);
+ throw t;
+ } finally {
+ if (blockReader != null) blockReader.close();
+ }
} catch (Throwable t) {
- LOG.error("error trying to retrieve a block reader " +
- "the second time.", t);
- throw t;
- } finally {
- if (blockReader != null) blockReader.close();
+ LOG.error("getBlockReader failure", t);
+ testFailed.set(true);
}
- } catch (Throwable t) {
- LOG.error("getBlockReader failure", t);
- testFailed.set(true);
}
+ };
+ Thread threads[] = new Thread[NUM_THREADS];
+ for (int i = 0; i < NUM_THREADS; i++) {
+ threads[i] = new Thread(readerRunnable);
+ threads[i].start();
}
- };
- Thread threads[] = new Thread[NUM_THREADS];
- for (int i = 0; i < NUM_THREADS; i++) {
- threads[i] = new Thread(readerRunnable);
- threads[i].start();
- }
- gotFailureLatch.await();
- replicaCreationShouldFail.set(false);
- shouldRetryLatch.countDown();
- for (int i = 0; i < NUM_THREADS; i++) {
- Uninterruptibles.joinUninterruptibly(threads[i]);
+ gotFailureLatch.await();
+ replicaCreationShouldFail.set(false);
+ shouldRetryLatch.countDown();
+ for (int i = 0; i < NUM_THREADS; i++) {
+ Uninterruptibles.joinUninterruptibly(threads[i]);
+ }
+ assertFalse(testFailed.get());
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
}
- cluster.shutdown();
- sockDir.close();
- assertFalse(testFailed.get());
}
/**
@@ -452,38 +461,41 @@ public void testShortCircuitReadFromServerWithoutShm() throws Exception {
DFSInputStream.tcpReadsDisabledForTesting = true;
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
- cluster.waitActive();
- clientConf.set(DFS_CLIENT_CONTEXT,
- "testShortCircuitReadFromServerWithoutShm_clientContext");
- final DistributedFileSystem fs =
- (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf);
- final String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 4000;
- final int SEED = 0xFADEC;
- DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
- byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
- assertTrue(Arrays.equals(contents, expected));
- final ShortCircuitCache cache =
- fs.getClient().getClientContext().getShortCircuitCache(0);
- final DatanodeInfo datanode = new DatanodeInfoBuilder()
- .setNodeID(cluster.getDataNodes().get(0).getDatanodeId())
- .build();
- cache.getDfsClientShmManager().visit(new Visitor() {
- @Override
- public void visit(HashMap info)
- throws IOException {
- assertEquals(1, info.size());
- PerDatanodeVisitorInfo vinfo = info.get(datanode);
- assertTrue(vinfo.disabled);
- assertEquals(0, vinfo.full.size());
- assertEquals(0, vinfo.notFull.size());
- }
- });
- cluster.shutdown();
- sockDir.close();
+ try {
+ cluster.waitActive();
+ clientConf.set(DFS_CLIENT_CONTEXT,
+ "testShortCircuitReadFromServerWithoutShm_clientContext");
+ final DistributedFileSystem fs =
+ (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf);
+ final String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 4000;
+ final int SEED = 0xFADEC;
+ DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ assertTrue(Arrays.equals(contents, expected));
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache(0);
+ final DatanodeInfo datanode = new DatanodeInfoBuilder()
+ .setNodeID(cluster.getDataNodes().get(0).getDatanodeId())
+ .build();
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap info)
+ throws IOException {
+ assertEquals(1, info.size());
+ PerDatanodeVisitorInfo vinfo = info.get(datanode);
+ assertTrue(vinfo.disabled);
+ assertEquals(0, vinfo.full.size());
+ assertEquals(0, vinfo.notFull.size());
+ }
+ });
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
/**
@@ -499,27 +511,30 @@ public void testShortCircuitReadFromClientWithoutShm() throws Exception {
DFSInputStream.tcpReadsDisabledForTesting = true;
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
- cluster.waitActive();
- clientConf.setInt(
- DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS, 0);
- clientConf.set(DFS_CLIENT_CONTEXT,
- "testShortCircuitReadFromClientWithoutShm_clientContext");
- final DistributedFileSystem fs =
- (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf);
- final String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 4000;
- final int SEED = 0xFADEC;
- DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
- byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
- assertTrue(Arrays.equals(contents, expected));
- final ShortCircuitCache cache =
- fs.getClient().getClientContext().getShortCircuitCache(0);
- assertEquals(null, cache.getDfsClientShmManager());
- cluster.shutdown();
- sockDir.close();
+ try {
+ cluster.waitActive();
+ clientConf.setInt(
+ DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS, 0);
+ clientConf.set(DFS_CLIENT_CONTEXT,
+ "testShortCircuitReadFromClientWithoutShm_clientContext");
+ final DistributedFileSystem fs =
+ (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf);
+ final String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 4000;
+ final int SEED = 0xFADEC;
+ DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ assertTrue(Arrays.equals(contents, expected));
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache(0);
+ assertEquals(null, cache.getDfsClientShmManager());
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
/**
@@ -535,25 +550,28 @@ public void testShortCircuitCacheShutdown() throws Exception {
DFSInputStream.tcpReadsDisabledForTesting = true;
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
- cluster.waitActive();
- final DistributedFileSystem fs =
- (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf);
- final String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 4000;
- final int SEED = 0xFADEC;
- DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
- byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
- assertTrue(Arrays.equals(contents, expected));
- final ShortCircuitCache cache =
- fs.getClient().getClientContext().getShortCircuitCache(0);
- cache.close();
- assertTrue(cache.getDfsClientShmManager().
- getDomainSocketWatcher().isClosed());
- cluster.shutdown();
- sockDir.close();
+ try {
+ cluster.waitActive();
+ final DistributedFileSystem fs =
+ (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf);
+ final String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 4000;
+ final int SEED = 0xFADEC;
+ DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE));
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ assertTrue(Arrays.equals(contents, expected));
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache(0);
+ cache.close();
+ assertTrue(cache.getDfsClientShmManager().
+ getDomainSocketWatcher().isClosed());
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
/**
@@ -593,84 +611,87 @@ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() {
"testPurgingClosedReplicas", sockDir);
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- final DistributedFileSystem dfs = cluster.getFileSystem();
- final String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 4095;
- final int SEED = 0xFADE0;
- final DistributedFileSystem fs =
- (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf);
- DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
-
- final Semaphore sem = new Semaphore(0);
- final List locatedBlocks =
- cluster.getNameNode().getRpcServer().getBlockLocations(
- TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks();
- final LocatedBlock lblock = locatedBlocks.get(0); // first block
- final byte[] buf = new byte[TEST_FILE_LEN];
- Runnable readerRunnable = new Runnable() {
- @Override
- public void run() {
- try {
- while (true) {
- BlockReader blockReader = null;
- try {
- blockReader = BlockReaderTestUtil.getBlockReader(
- cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
- sem.release();
+ try {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 4095;
+ final int SEED = 0xFADE0;
+ final DistributedFileSystem fs =
+ (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf);
+ DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+
+ final Semaphore sem = new Semaphore(0);
+ final List locatedBlocks =
+ cluster.getNameNode().getRpcServer().getBlockLocations(
+ TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks();
+ final LocatedBlock lblock = locatedBlocks.get(0); // first block
+ final byte[] buf = new byte[TEST_FILE_LEN];
+ Runnable readerRunnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ BlockReader blockReader = null;
try {
- blockReader.readAll(buf, 0, TEST_FILE_LEN);
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
+ sem.release();
+ try {
+ blockReader.readAll(buf, 0, TEST_FILE_LEN);
+ } finally {
+ sem.acquireUninterruptibly();
+ }
+ } catch (ClosedByInterruptException e) {
+ LOG.info("got the expected ClosedByInterruptException", e);
+ sem.release();
+ break;
} finally {
- sem.acquireUninterruptibly();
+ if (blockReader != null) blockReader.close();
}
- } catch (ClosedByInterruptException e) {
- LOG.info("got the expected ClosedByInterruptException", e);
- sem.release();
- break;
- } finally {
- if (blockReader != null) blockReader.close();
+ LOG.info("read another " + TEST_FILE_LEN + " bytes.");
}
- LOG.info("read another " + TEST_FILE_LEN + " bytes.");
+ } catch (Throwable t) {
+ LOG.error("getBlockReader failure", t);
+ testFailed.set(true);
+ sem.release();
}
- } catch (Throwable t) {
- LOG.error("getBlockReader failure", t);
- testFailed.set(true);
- sem.release();
}
+ };
+ Thread thread = new Thread(readerRunnable);
+ thread.start();
+
+ // While the thread is reading, send it interrupts.
+ // These should trigger a ClosedChannelException.
+ while (thread.isAlive()) {
+ sem.acquireUninterruptibly();
+ thread.interrupt();
+ sem.release();
}
- };
- Thread thread = new Thread(readerRunnable);
- thread.start();
-
- // While the thread is reading, send it interrupts.
- // These should trigger a ClosedChannelException.
- while (thread.isAlive()) {
- sem.acquireUninterruptibly();
- thread.interrupt();
- sem.release();
- }
- assertFalse(testFailed.get());
+ assertFalse(testFailed.get());
+
+ // We should be able to read from the file without
+ // getting a ClosedChannelException.
+ BlockReader blockReader = null;
+ try {
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
+ blockReader.readFully(buf, 0, TEST_FILE_LEN);
+ } finally {
+ if (blockReader != null) blockReader.close();
+ }
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ assertTrue(Arrays.equals(buf, expected));
- // We should be able to read from the file without
- // getting a ClosedChannelException.
- BlockReader blockReader = null;
- try {
- blockReader = BlockReaderTestUtil.getBlockReader(
- cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
- blockReader.readFully(buf, 0, TEST_FILE_LEN);
+ // Another ShortCircuitReplica object should have been created.
+ assertEquals(2, replicasCreated.get());
+
+ dfs.close();
} finally {
- if (blockReader != null) blockReader.close();
+ cluster.shutdown();
+ sockDir.close();
}
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
- assertTrue(Arrays.equals(buf, expected));
-
- // Another ShortCircuitReplica object should have been created.
- assertEquals(2, replicasCreated.get());
-
- dfs.close();
- cluster.shutdown();
- sockDir.close();
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java
index 88a1f8be3b2585..dc531fbc1f9774 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java
@@ -98,45 +98,48 @@ public void testStablePositionAfterCorruptRead() throws Exception {
File basedir = new File(GenericTestUtils.getRandomizedTempPath());
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf, basedir).numDataNodes(1).build();
- cluster.waitActive();
- FileSystem fs = cluster.getFileSystem();
-
- Path path = new Path("/corrupted");
-
- DFSTestUtil.createFile(fs, path, FILE_LENGTH, REPL_FACTOR, 12345L);
- DFSTestUtil.waitReplication(fs, path, REPL_FACTOR);
-
- ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, path);
- int blockFilesCorrupted = cluster.corruptBlockOnDataNodes(block);
- assertEquals(REPL_FACTOR, blockFilesCorrupted, "All replicas not corrupted");
-
- FSDataInputStream dis = cluster.getFileSystem().open(path);
- ByteBuffer buf = ByteBuffer.allocateDirect((int)FILE_LENGTH);
- boolean sawException = false;
- try {
- dis.read(buf);
- } catch (ChecksumException ex) {
- sawException = true;
- }
-
- assertTrue(sawException);
- assertEquals(0, buf.position());
- assertEquals(buf.capacity(), buf.limit());
-
- dis = cluster.getFileSystem().open(path);
- buf.position(3);
- buf.limit(25);
- sawException = false;
try {
- dis.read(buf);
- } catch (ChecksumException ex) {
- sawException = true;
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+
+ Path path = new Path("/corrupted");
+
+ DFSTestUtil.createFile(fs, path, FILE_LENGTH, REPL_FACTOR, 12345L);
+ DFSTestUtil.waitReplication(fs, path, REPL_FACTOR);
+
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, path);
+ int blockFilesCorrupted = cluster.corruptBlockOnDataNodes(block);
+ assertEquals(REPL_FACTOR, blockFilesCorrupted, "All replicas not corrupted");
+
+ FSDataInputStream dis = cluster.getFileSystem().open(path);
+ ByteBuffer buf = ByteBuffer.allocateDirect((int)FILE_LENGTH);
+ boolean sawException = false;
+ try {
+ dis.read(buf);
+ } catch (ChecksumException ex) {
+ sawException = true;
+ }
+
+ assertTrue(sawException);
+ assertEquals(0, buf.position());
+ assertEquals(buf.capacity(), buf.limit());
+
+ dis = cluster.getFileSystem().open(path);
+ buf.position(3);
+ buf.limit(25);
+ sawException = false;
+ try {
+ dis.read(buf);
+ } catch (ChecksumException ex) {
+ sawException = true;
+ }
+
+ assertTrue(sawException);
+ assertEquals(3, buf.position());
+ assertEquals(25, buf.limit());
+ } finally {
+ cluster.shutdown();
}
-
- assertTrue(sawException);
- assertEquals(3, buf.position());
- assertEquals(25, buf.limit());
- cluster.shutdown();
}
@Test
@@ -148,26 +151,29 @@ public void testBothOldAndNewShortCircuitConfigured() throws Exception {
HdfsConfiguration conf = getConfiguration(socketDir);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- socketDir.close();
- FileSystem fs = cluster.getFileSystem();
-
- Path path = new Path("/foo");
- byte orig[] = new byte[FILE_LENGTH];
- for (int i = 0; i < orig.length; i++) {
- orig[i] = (byte)(i%10);
+ try {
+ cluster.waitActive();
+ socketDir.close();
+ FileSystem fs = cluster.getFileSystem();
+
+ Path path = new Path("/foo");
+ byte orig[] = new byte[FILE_LENGTH];
+ for (int i = 0; i < orig.length; i++) {
+ orig[i] = (byte)(i%10);
+ }
+ FSDataOutputStream fos = fs.create(path, (short)1);
+ fos.write(orig);
+ fos.close();
+ DFSTestUtil.waitReplication(fs, path, REPL_FACTOR);
+ FSDataInputStream fis = cluster.getFileSystem().open(path);
+ byte buf[] = new byte[FILE_LENGTH];
+ IOUtils.readFully(fis, buf, 0, FILE_LENGTH);
+ fis.close();
+ assertArrayEquals(orig, buf);
+ Arrays.equals(orig, buf);
+ } finally {
+ cluster.shutdown();
}
- FSDataOutputStream fos = fs.create(path, (short)1);
- fos.write(orig);
- fos.close();
- DFSTestUtil.waitReplication(fs, path, REPL_FACTOR);
- FSDataInputStream fis = cluster.getFileSystem().open(path);
- byte buf[] = new byte[FILE_LENGTH];
- IOUtils.readFully(fis, buf, 0, FILE_LENGTH);
- fis.close();
- assertArrayEquals(orig, buf);
- Arrays.equals(orig, buf);
- cluster.shutdown();
}
@Test
@@ -180,53 +186,56 @@ public void testBlockReaderLocalLegacyWithAppend() throws Exception {
File basedir = new File(GenericTestUtils.getRandomizedTempPath());
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf, basedir).numDataNodes(1).build();
- cluster.waitActive();
-
- final DistributedFileSystem dfs = cluster.getFileSystem();
- final Path path = new Path("/testBlockReaderLocalLegacy");
- DFSTestUtil.createFile(dfs, path, 10, REPL_FACTOR, 0);
- DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR);
-
- final ClientDatanodeProtocol proxy;
- final Token token;
- final ExtendedBlock originalBlock;
- final long originalGS;
- {
- final LocatedBlock lb = cluster.getNameNode().getRpcServer()
- .getBlockLocations(path.toString(), 0, 1).get(0);
- proxy = DFSUtilClient.createClientDatanodeProtocolProxy(
- lb.getLocations()[0], conf, 60000, false);
- token = lb.getBlockToken();
-
- // get block and generation stamp
- final ExtendedBlock blk = new ExtendedBlock(lb.getBlock());
- originalBlock = new ExtendedBlock(blk);
- originalGS = originalBlock.getGenerationStamp();
-
- // test getBlockLocalPathInfo
- final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(blk, token);
- assertEquals(originalGS, info.getBlock().getGenerationStamp());
- }
-
- { // append one byte
- FSDataOutputStream out = dfs.append(path);
- out.write(1);
- out.close();
- }
-
- {
- // get new generation stamp
- final LocatedBlock lb = cluster.getNameNode().getRpcServer()
- .getBlockLocations(path.toString(), 0, 1).get(0);
- final long newGS = lb.getBlock().getGenerationStamp();
- assertTrue(newGS > originalGS);
-
- // getBlockLocalPathInfo using the original block.
- assertEquals(originalGS, originalBlock.getGenerationStamp());
- final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(
- originalBlock, token);
- assertEquals(newGS, info.getBlock().getGenerationStamp());
+ try {
+ cluster.waitActive();
+
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final Path path = new Path("/testBlockReaderLocalLegacy");
+ DFSTestUtil.createFile(dfs, path, 10, REPL_FACTOR, 0);
+ DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR);
+
+ final ClientDatanodeProtocol proxy;
+ final Token token;
+ final ExtendedBlock originalBlock;
+ final long originalGS;
+ {
+ final LocatedBlock lb = cluster.getNameNode().getRpcServer()
+ .getBlockLocations(path.toString(), 0, 1).get(0);
+ proxy = DFSUtilClient.createClientDatanodeProtocolProxy(
+ lb.getLocations()[0], conf, 60000, false);
+ token = lb.getBlockToken();
+
+ // get block and generation stamp
+ final ExtendedBlock blk = new ExtendedBlock(lb.getBlock());
+ originalBlock = new ExtendedBlock(blk);
+ originalGS = originalBlock.getGenerationStamp();
+
+ // test getBlockLocalPathInfo
+ final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(blk, token);
+ assertEquals(originalGS, info.getBlock().getGenerationStamp());
+ }
+
+ { // append one byte
+ FSDataOutputStream out = dfs.append(path);
+ out.write(1);
+ out.close();
+ }
+
+ {
+ // get new generation stamp
+ final LocatedBlock lb = cluster.getNameNode().getRpcServer()
+ .getBlockLocations(path.toString(), 0, 1).get(0);
+ final long newGS = lb.getBlock().getGenerationStamp();
+ assertTrue(newGS > originalGS);
+
+ // getBlockLocalPathInfo using the original block.
+ assertEquals(originalGS, originalBlock.getGenerationStamp());
+ final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(
+ originalBlock, token);
+ assertEquals(newGS, info.getBlock().getGenerationStamp());
+ }
+ } finally {
+ cluster.shutdown();
}
- cluster.shutdown();
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
index 298c856655dda6..8f1c0de829f15b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java
@@ -36,12 +36,16 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TestNNWithQJM {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestNNWithQJM.class);
final Configuration conf = new HdfsConfiguration();
private MiniJournalCluster mjc = null;
private final Path TEST_PATH = new Path("/test-dir");
@@ -168,7 +172,19 @@ public void testNewNamenodeTakesOverWriter() throws Exception {
"Could not sync enough journals to persistent storage", re);
}
} finally {
- //cluster.shutdown();
+ if (cluster != null) {
+ try {
+ cluster.shutdown();
+ } catch (ExitUtil.ExitException e) {
+ // Expected: this NN was fenced by the second cluster above, so
+ // closing its edit log cannot reach a journal quorum and the
+ // shutdown terminates with "Could not sync enough journals".
+ // Releasing what can be released still beats leaving the NN
+ // running for the remaining tests in this class.
+ LOG.warn("Expected exit while shutting down the fenced NN", e);
+ ExitUtil.resetFirstExitException();
+ }
+ }
}
}
@@ -188,23 +204,34 @@ public void testMismatchedNNIsRejected() throws Exception {
.manageNameDfsDirs(false)
.build();
cluster.shutdown();
-
- // Reformat just the on-disk portion
- Configuration onDiskOnly = new Configuration(conf);
- onDiskOnly.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, defaultEditsDir);
- NameNode.format(onDiskOnly);
+ // Null out the reference: the build below is expected to throw before
+ // reassigning it, and the finally must not shut this cluster down a
+ // second time -- that would re-run the ExitUtil exit check, whose
+ // AssertionError could mask the test's real failure.
+ cluster = null;
- // Start the NN - should fail because the JNs are still formatted
- // with the old namespace ID.
try {
- ExitUtil.disableSystemExit();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
- .manageNameDfsDirs(false).format(false).checkExitOnShutdown(false)
- .build();
- fail("New NN with different namespace should have been rejected");
- } catch (IOException ioe) {
- GenericTestUtils.assertExceptionContains(
- "recoverUnfinalizedSegments failed for too many journals", ioe);
+ // Reformat just the on-disk portion
+ Configuration onDiskOnly = new Configuration(conf);
+ onDiskOnly.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, defaultEditsDir);
+ NameNode.format(onDiskOnly);
+
+ // Start the NN - should fail because the JNs are still formatted
+ // with the old namespace ID.
+ try {
+ ExitUtil.disableSystemExit();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
+ .manageNameDfsDirs(false).format(false).checkExitOnShutdown(false)
+ .build();
+ fail("New NN with different namespace should have been rejected");
+ } catch (IOException ioe) {
+ GenericTestUtils.assertExceptionContains(
+ "recoverUnfinalizedSegments failed for too many journals", ioe);
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
index 1ebb475cf7708a..982cac70d5e288 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
@@ -41,7 +41,6 @@
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
-import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -106,13 +105,15 @@ public static void waitStoragesNoStale(MiniDFSCluster cluster,
* datanodes); It then adds one new empty node and starts balancing.
*/
@Test
- @Timeout(value = 60)
+ @Timeout(value = 300)
public void testBalancerWithHANameNodes() throws Exception {
+ // 300s, not 60s: waitStoragesNoStale alone may legitimately take up
+ // to 60s, so a 60s test budget fires before the balancer even runs
+ // on a loaded CI agent. The observer variants below already use
+ // 120s/180s for the same work.
Configuration conf = new HdfsConfiguration();
TestBalancer.initConf(conf);
assertEquals(TEST_CAPACITIES.length, TEST_RACKS.length);
- NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1");
- nn1Conf.setIpcPort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
Configuration copiedConf = new Configuration(conf);
cluster = new MiniDFSCluster.Builder(copiedConf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
@@ -181,7 +182,7 @@ void doTest(Configuration conf, boolean withHA) throws Exception {
* Test Balancer request Standby NameNode when enable this feature.
*/
@Test
- @Timeout(value = 60)
+ @Timeout(value = 300)
public void testBalancerRequestSBNWithHA() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setBoolean(DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY, false);
@@ -189,8 +190,6 @@ public void testBalancerRequestSBNWithHA() throws Exception {
//conf.setBoolean(DFS_HA_BALANCER_REQUEST_STANDBY_KEY, true);
TestBalancer.initConf(conf);
assertEquals(TEST_CAPACITIES.length, TEST_RACKS.length);
- NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1");
- nn1Conf.setIpcPort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
Configuration copiedConf = new Configuration(conf);
cluster = new MiniDFSCluster.Builder(copiedConf)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
@@ -302,8 +301,6 @@ public void testGetLiveDatanodeStorageReport() throws Exception {
Configuration conf = new HdfsConfiguration();
TestBalancer.initConf(conf);
assertEquals(TEST_CAPACITIES.length, TEST_RACKS.length);
- NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1");
- nn1Conf.setIpcPort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
Configuration copiedConf = new Configuration(conf);
// Try capture NameNodeConnector log.
LogCapturer log =LogCapturer.captureLogs(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java
index 95f97d1a93eb3f..9c0071d2d0eef2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java
@@ -138,28 +138,31 @@ public void removeBlockReportLease(DatanodeDescriptor node, long leaseId) {
final int NUM_DATANODES = 5;
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
- cluster.waitActive();
- for (int n = 1; n <= NUM_DATANODES; n++) {
- LOG.info("Waiting for " + n + " datanode(s) to report in.");
- fbrSem.release();
- Uninterruptibles.sleepUninterruptibly(20, TimeUnit.MILLISECONDS);
- final int currentN = n;
- GenericTestUtils.waitFor(new Supplier() {
- @Override
- public Boolean get() {
- synchronized (injector) {
- if (fbrDns.size() > currentN) {
- setFailure(failure, "Expected at most " + currentN +
- " datanodes to have sent a block report, but actually " +
- fbrDns.size() + " have.");
+ try {
+ cluster.waitActive();
+ for (int n = 1; n <= NUM_DATANODES; n++) {
+ LOG.info("Waiting for " + n + " datanode(s) to report in.");
+ fbrSem.release();
+ Uninterruptibles.sleepUninterruptibly(20, TimeUnit.MILLISECONDS);
+ final int currentN = n;
+ GenericTestUtils.waitFor(new Supplier() {
+ @Override
+ public Boolean get() {
+ synchronized (injector) {
+ if (fbrDns.size() > currentN) {
+ setFailure(failure, "Expected at most " + currentN +
+ " datanodes to have sent a block report, but actually " +
+ fbrDns.size() + " have.");
+ }
+ return (fbrDns.size() >= currentN);
}
- return (fbrDns.size() >= currentN);
}
- }
- }, 25, 50000);
+ }, 25, 50000);
+ }
+ assertEquals("", failure.get());
+ } finally {
+ cluster.shutdown();
}
- cluster.shutdown();
- assertEquals("", failure.get());
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java
index 7e7a8b6128a3f2..0c736d012e2ede 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.eclipse.jetty.util.ajax.JSON;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -73,6 +74,14 @@ public void setup() throws Exception {
fs = cluster.getFileSystem();
}
+ @AfterEach
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
/**
* Create files of different sizes for each datanode.
* Ensure that the file size is smaller than the blocksize
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java
index ececeb07e81419..8d8d8bffe12398 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java
@@ -58,71 +58,74 @@ public void testBlockCounters() throws Exception {
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- final DatanodeProtocolClientSideTranslatorPB spy =
- InternalDataNodeTestUtils.spyOnBposToNN(
- cluster.getDataNodes().get(0), cluster.getNameNode());
- final DataNode datanode = cluster.getDataNodes().get(0);
-
- /* We should get 0 incremental block report. */
- Mockito.verify(spy, timeout(60000).times(0)).blockReceivedAndDeleted(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageReceivedDeletedBlocks[].class));
-
- /*
- * Create fake blocks notification on the DataNode. This will be sent with
- * the next incremental block report.
- */
- final BPServiceActor actor =
- datanode.getAllBpOs().get(0).getBPServiceActors().get(0);
- final FsDatasetSpi dataset = datanode.getFSDataset();
- final DatanodeStorage storage;
- try (FsDatasetSpi.FsVolumeReferences volumes =
- dataset.getFsVolumeReferences()) {
- storage = dataset.getStorage(volumes.get(0).getStorageID());
+ try {
+ cluster.waitActive();
+ final DatanodeProtocolClientSideTranslatorPB spy =
+ InternalDataNodeTestUtils.spyOnBposToNN(
+ cluster.getDataNodes().get(0), cluster.getNameNode());
+ final DataNode datanode = cluster.getDataNodes().get(0);
+
+ /* We should get 0 incremental block report. */
+ Mockito.verify(spy, timeout(60000).times(0)).blockReceivedAndDeleted(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageReceivedDeletedBlocks[].class));
+
+ /*
+ * Create fake blocks notification on the DataNode. This will be sent with
+ * the next incremental block report.
+ */
+ final BPServiceActor actor =
+ datanode.getAllBpOs().get(0).getBPServiceActors().get(0);
+ final FsDatasetSpi dataset = datanode.getFSDataset();
+ final DatanodeStorage storage;
+ try (FsDatasetSpi.FsVolumeReferences volumes =
+ dataset.getFsVolumeReferences()) {
+ storage = dataset.getStorage(volumes.get(0).getStorageID());
+ }
+
+ ReceivedDeletedBlockInfo rdbi = null;
+ /* block at status of RECEIVING_BLOCK */
+ rdbi = new ReceivedDeletedBlockInfo(
+ new Block(5678, 512, 1000), BlockStatus.RECEIVING_BLOCK, null);
+ actor.getIbrManager().addRDBI(rdbi, storage);
+
+ /* block at status of RECEIVED_BLOCK */
+ rdbi = new ReceivedDeletedBlockInfo(
+ new Block(5679, 512, 1000), BlockStatus.RECEIVED_BLOCK, null);
+ actor.getIbrManager().addRDBI(rdbi, storage);
+
+ /* block at status of DELETED_BLOCK */
+ rdbi = new ReceivedDeletedBlockInfo(
+ new Block(5680, 512, 1000), BlockStatus.DELETED_BLOCK, null);
+ actor.getIbrManager().addRDBI(rdbi, storage);
+
+ /* verify counters before sending IBR */
+ verifyBlockCounters(datanode, 3, 1, 1, 1);
+
+ /* Manually trigger a block report. */
+ datanode.triggerBlockReport(
+ new BlockReportOptions.Factory().
+ setIncremental(true).
+ build()
+ );
+
+ /*
+ * triggerBlockReport returns before the block report is actually sent. Wait
+ * for it to be sent here.
+ */
+ Mockito.verify(spy, timeout(60000).times(1)).
+ blockReceivedAndDeleted(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageReceivedDeletedBlocks[].class));
+
+ /* verify counters after sending IBR */
+ verifyBlockCounters(datanode, 0, 0, 0, 0);
+
+ } finally {
+ cluster.shutdown();
}
-
- ReceivedDeletedBlockInfo rdbi = null;
- /* block at status of RECEIVING_BLOCK */
- rdbi = new ReceivedDeletedBlockInfo(
- new Block(5678, 512, 1000), BlockStatus.RECEIVING_BLOCK, null);
- actor.getIbrManager().addRDBI(rdbi, storage);
-
- /* block at status of RECEIVED_BLOCK */
- rdbi = new ReceivedDeletedBlockInfo(
- new Block(5679, 512, 1000), BlockStatus.RECEIVED_BLOCK, null);
- actor.getIbrManager().addRDBI(rdbi, storage);
-
- /* block at status of DELETED_BLOCK */
- rdbi = new ReceivedDeletedBlockInfo(
- new Block(5680, 512, 1000), BlockStatus.DELETED_BLOCK, null);
- actor.getIbrManager().addRDBI(rdbi, storage);
-
- /* verify counters before sending IBR */
- verifyBlockCounters(datanode, 3, 1, 1, 1);
-
- /* Manually trigger a block report. */
- datanode.triggerBlockReport(
- new BlockReportOptions.Factory().
- setIncremental(true).
- build()
- );
-
- /*
- * triggerBlockReport returns before the block report is actually sent. Wait
- * for it to be sent here.
- */
- Mockito.verify(spy, timeout(60000).times(1)).
- blockReceivedAndDeleted(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageReceivedDeletedBlocks[].class));
-
- /* verify counters after sending IBR */
- verifyBlockCounters(datanode, 0, 0, 0, 0);
-
- cluster.shutdown();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java
index ff403c65310544..1e261c6a02ba5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java
@@ -78,9 +78,9 @@ public void testDataNodeInitStorage() throws Throwable {
// Start a cluster so that SimulatedFsDatasetVerifier constructor is
// invoked.
- MiniDFSCluster cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- cluster.shutdown();
+ try (MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) {
+ cluster.waitActive();
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
index bf72ea9ede52c2..4329644e10c24f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
@@ -606,11 +606,12 @@ public void testNNRpcMetricsWithNonHA() throws IOException {
// heartbeat periodically to NN during running test case, and bpServiceActor
// only sends heartbeat once after startup
conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- cluster.waitActive();
- DataNode dn = cluster.getDataNodes().get(0);
- MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
- assertCounter("HeartbeatsNumOps", 1L, rb);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+ cluster.waitActive();
+ DataNode dn = cluster.getDataNodes().get(0);
+ MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
+ assertCounter("HeartbeatsNumOps", 1L, rb);
+ }
}
@Test
@@ -676,15 +677,16 @@ public void testNNRpcMetricsWithHA() throws IOException {
// heartbeat periodically to NN during running test case, and bpServiceActor
// only sends heartbeat once after startup
conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
- MiniDFSNNTopology.simpleHATopology()).build();
- cluster.waitActive();
- DataNode dn = cluster.getDataNodes().get(0);
- cluster.transitionToActive(0);
- MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
- assertCounter("HeartbeatsForminidfs-ns-nn1NumOps", 1L, rb);
- assertCounter("HeartbeatsForminidfs-ns-nn2NumOps", 1L, rb);
- assertCounter("HeartbeatsNumOps", 2L, rb);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
+ MiniDFSNNTopology.simpleHATopology()).build()) {
+ cluster.waitActive();
+ DataNode dn = cluster.getDataNodes().get(0);
+ cluster.transitionToActive(0);
+ MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
+ assertCounter("HeartbeatsForminidfs-ns-nn1NumOps", 1L, rb);
+ assertCounter("HeartbeatsForminidfs-ns-nn2NumOps", 1L, rb);
+ assertCounter("HeartbeatsNumOps", 2L, rb);
+ }
}
@Test
@@ -694,14 +696,15 @@ public void testNNRpcMetricsWithFederation() throws IOException {
// heartbeat periodically to NN during running test case, and bpServiceActor
// only sends heartbeat once after startup
conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
- MiniDFSNNTopology.simpleFederatedTopology("ns1,ns2")).build();
- cluster.waitActive();
- DataNode dn = cluster.getDataNodes().get(0);
- MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
- assertCounter("HeartbeatsForns1NumOps", 1L, rb);
- assertCounter("HeartbeatsForns2NumOps", 1L, rb);
- assertCounter("HeartbeatsNumOps", 2L, rb);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
+ MiniDFSNNTopology.simpleFederatedTopology("ns1,ns2")).build()) {
+ cluster.waitActive();
+ DataNode dn = cluster.getDataNodes().get(0);
+ MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
+ assertCounter("HeartbeatsForns1NumOps", 1L, rb);
+ assertCounter("HeartbeatsForns2NumOps", 1L, rb);
+ assertCounter("HeartbeatsNumOps", 2L, rb);
+ }
}
@Test
@@ -711,17 +714,18 @@ public void testNNRpcMetricsWithFederationAndHA() throws IOException {
// heartbeat periodically to NN during running test case, and bpServiceActor
// only sends heartbeat once after startup
conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
- MiniDFSNNTopology.simpleHAFederatedTopology(2)).build();
- cluster.waitActive();
- DataNode dn = cluster.getDataNodes().get(0);
- MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
-
- assertCounter("HeartbeatsForns0-nn0NumOps", 1L, rb);
- assertCounter("HeartbeatsForns0-nn1NumOps", 1L, rb);
- assertCounter("HeartbeatsForns1-nn0NumOps", 1L, rb);
- assertCounter("HeartbeatsForns1-nn1NumOps", 1L, rb);
- assertCounter("HeartbeatsNumOps", 4L, rb);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
+ MiniDFSNNTopology.simpleHAFederatedTopology(2)).build()) {
+ cluster.waitActive();
+ DataNode dn = cluster.getDataNodes().get(0);
+ MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name());
+
+ assertCounter("HeartbeatsForns0-nn0NumOps", 1L, rb);
+ assertCounter("HeartbeatsForns0-nn1NumOps", 1L, rb);
+ assertCounter("HeartbeatsForns1-nn0NumOps", 1L, rb);
+ assertCounter("HeartbeatsForns1-nn1NumOps", 1L, rb);
+ assertCounter("HeartbeatsNumOps", 4L, rb);
+ }
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java
index 46dd31c31e2f11..480c0c5885bbcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java
@@ -64,56 +64,59 @@ public void testHSyncWithAppend() throws Exception {
private void testHSyncOperation(boolean testWithAppend) throws IOException {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- final DistributedFileSystem fs = cluster.getFileSystem();
+ try {
+ final DistributedFileSystem fs = cluster.getFileSystem();
- final Path p = new Path("/testHSync/foo");
- final int len = 1 << 16;
- FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
- 4096, (short) 1, len, null);
- if (testWithAppend) {
- // re-open the file with append call
+ final Path p = new Path("/testHSync/foo");
+ final int len = 1 << 16;
+ FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
+ 4096, (short) 1, len, null);
+ if (testWithAppend) {
+ // re-open the file with append call
+ out.close();
+ out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.SYNC_BLOCK),
+ 4096, null);
+ }
+ out.hflush();
+ // hflush does not sync
+ checkSyncMetric(cluster, 0);
+ out.hsync();
+ // hsync on empty file does nothing
+ checkSyncMetric(cluster, 0);
+ out.write(1);
+ checkSyncMetric(cluster, 0);
+ out.hsync();
+ checkSyncMetric(cluster, 1);
+ // avoiding repeated hsyncs is a potential future optimization
+ out.hsync();
+ checkSyncMetric(cluster, 2);
+ out.hflush();
+ // hflush still does not sync
+ checkSyncMetric(cluster, 2);
out.close();
- out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.SYNC_BLOCK),
- 4096, null);
- }
- out.hflush();
- // hflush does not sync
- checkSyncMetric(cluster, 0);
- out.hsync();
- // hsync on empty file does nothing
- checkSyncMetric(cluster, 0);
- out.write(1);
- checkSyncMetric(cluster, 0);
- out.hsync();
- checkSyncMetric(cluster, 1);
- // avoiding repeated hsyncs is a potential future optimization
- out.hsync();
- checkSyncMetric(cluster, 2);
- out.hflush();
- // hflush still does not sync
- checkSyncMetric(cluster, 2);
- out.close();
- // close is sync'ing
- checkSyncMetric(cluster, 3);
+ // close is sync'ing
+ checkSyncMetric(cluster, 3);
- // same with a file created with out SYNC_BLOCK
- out = fs.create(p, FsPermission.getDefault(),
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
- 4096, (short) 1, len, null);
- out.hsync();
- checkSyncMetric(cluster, 3);
- out.write(1);
- checkSyncMetric(cluster, 3);
- out.hsync();
- checkSyncMetric(cluster, 4);
- // repeated hsyncs
- out.hsync();
- checkSyncMetric(cluster, 5);
- out.close();
- // close does not sync (not opened with SYNC_BLOCK)
- checkSyncMetric(cluster, 5);
- cluster.shutdown();
+ // same with a file created with out SYNC_BLOCK
+ out = fs.create(p, FsPermission.getDefault(),
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
+ 4096, (short) 1, len, null);
+ out.hsync();
+ checkSyncMetric(cluster, 3);
+ out.write(1);
+ checkSyncMetric(cluster, 3);
+ out.hsync();
+ checkSyncMetric(cluster, 4);
+ // repeated hsyncs
+ out.hsync();
+ checkSyncMetric(cluster, 5);
+ out.close();
+ // close does not sync (not opened with SYNC_BLOCK)
+ checkSyncMetric(cluster, 5);
+ } finally {
+ cluster.shutdown();
+ }
}
/** Test hsync on an exact block boundary */
@@ -121,29 +124,32 @@ private void testHSyncOperation(boolean testWithAppend) throws IOException {
public void testHSyncBlockBoundary() throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- final FileSystem fs = cluster.getFileSystem();
+ try {
+ final FileSystem fs = cluster.getFileSystem();
- final Path p = new Path("/testHSyncBlockBoundary/foo");
- final int len = 1 << 16;
- final byte[] fileContents = AppendTestUtil.initBuffer(len);
- FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
- 4096, (short) 1, len, null);
- // fill exactly one block (tests the SYNC_BLOCK case) and flush
- out.write(fileContents, 0, len);
- out.hflush();
- // the full block should have caused a sync
- checkSyncMetric(cluster, 1);
- out.hsync();
- // first on block again
- checkSyncMetric(cluster, 1);
- // write one more byte and sync again
- out.write(1);
- out.hsync();
- checkSyncMetric(cluster, 2);
- out.close();
- checkSyncMetric(cluster, 3);
- cluster.shutdown();
+ final Path p = new Path("/testHSyncBlockBoundary/foo");
+ final int len = 1 << 16;
+ final byte[] fileContents = AppendTestUtil.initBuffer(len);
+ FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
+ 4096, (short) 1, len, null);
+ // fill exactly one block (tests the SYNC_BLOCK case) and flush
+ out.write(fileContents, 0, len);
+ out.hflush();
+ // the full block should have caused a sync
+ checkSyncMetric(cluster, 1);
+ out.hsync();
+ // first on block again
+ checkSyncMetric(cluster, 1);
+ // write one more byte and sync again
+ out.write(1);
+ out.hsync();
+ checkSyncMetric(cluster, 2);
+ out.close();
+ checkSyncMetric(cluster, 3);
+ } finally {
+ cluster.shutdown();
+ }
}
/** Test hsync via SequenceFiles */
@@ -151,33 +157,36 @@ public void testHSyncBlockBoundary() throws Exception {
public void testSequenceFileSync() throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ try {
- final FileSystem fs = cluster.getFileSystem();
- final Path p = new Path("/testSequenceFileSync/foo");
- final int len = 1 << 16;
- FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
- 4096, (short) 1, len, null);
- Writer w = SequenceFile.createWriter(new Configuration(),
- Writer.stream(out),
- Writer.keyClass(RandomDatum.class),
- Writer.valueClass(RandomDatum.class),
- Writer.compression(CompressionType.NONE, new DefaultCodec()));
- w.hflush();
- checkSyncMetric(cluster, 0);
- w.hsync();
- checkSyncMetric(cluster, 1);
- int seed = new Random().nextInt();
- RandomDatum.Generator generator = new RandomDatum.Generator(seed);
- generator.next();
- w.append(generator.getKey(), generator.getValue());
- w.hsync();
- checkSyncMetric(cluster, 2);
- w.close();
- checkSyncMetric(cluster, 2);
- out.close();
- checkSyncMetric(cluster, 3);
- cluster.shutdown();
+ final FileSystem fs = cluster.getFileSystem();
+ final Path p = new Path("/testSequenceFileSync/foo");
+ final int len = 1 << 16;
+ FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
+ 4096, (short) 1, len, null);
+ Writer w = SequenceFile.createWriter(new Configuration(),
+ Writer.stream(out),
+ Writer.keyClass(RandomDatum.class),
+ Writer.valueClass(RandomDatum.class),
+ Writer.compression(CompressionType.NONE, new DefaultCodec()));
+ w.hflush();
+ checkSyncMetric(cluster, 0);
+ w.hsync();
+ checkSyncMetric(cluster, 1);
+ int seed = new Random().nextInt();
+ RandomDatum.Generator generator = new RandomDatum.Generator(seed);
+ generator.next();
+ w.append(generator.getKey(), generator.getValue());
+ w.hsync();
+ checkSyncMetric(cluster, 2);
+ w.close();
+ checkSyncMetric(cluster, 2);
+ out.close();
+ checkSyncMetric(cluster, 3);
+ } finally {
+ cluster.shutdown();
+ }
}
/** Test that syncBlock is correctly performed at replicas */
@@ -185,26 +194,29 @@ public void testSequenceFileSync() throws Exception {
public void testHSyncWithReplication() throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
- final FileSystem fs = cluster.getFileSystem();
+ try {
+ final FileSystem fs = cluster.getFileSystem();
- final Path p = new Path("/testHSyncWithReplication/foo");
- final int len = 1 << 16;
- FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
- EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
- 4096, (short) 3, len, null);
- out.write(1);
- out.hflush();
- checkSyncMetric(cluster, 0, 0);
- checkSyncMetric(cluster, 1, 0);
- checkSyncMetric(cluster, 2, 0);
- out.hsync();
- checkSyncMetric(cluster, 0, 1);
- checkSyncMetric(cluster, 1, 1);
- checkSyncMetric(cluster, 2, 1);
- out.hsync();
- checkSyncMetric(cluster, 0, 2);
- checkSyncMetric(cluster, 1, 2);
- checkSyncMetric(cluster, 2, 2);
- cluster.shutdown();
+ final Path p = new Path("/testHSyncWithReplication/foo");
+ final int len = 1 << 16;
+ FSDataOutputStream out = fs.create(p, FsPermission.getDefault(),
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK),
+ 4096, (short) 3, len, null);
+ out.write(1);
+ out.hflush();
+ checkSyncMetric(cluster, 0, 0);
+ checkSyncMetric(cluster, 1, 0);
+ checkSyncMetric(cluster, 2, 0);
+ out.hsync();
+ checkSyncMetric(cluster, 0, 1);
+ checkSyncMetric(cluster, 1, 1);
+ checkSyncMetric(cluster, 2, 1);
+ out.hsync();
+ checkSyncMetric(cluster, 0, 2);
+ checkSyncMetric(cluster, 1, 2);
+ checkSyncMetric(cluster, 2, 2);
+ } finally {
+ cluster.shutdown();
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java
index c2ecaf1d246869..e78ec230eb0f86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java
@@ -61,104 +61,107 @@ private void testTriggerBlockReport(boolean incremental, boolean withSpecificNN)
final MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(1).build();
- cluster.waitActive();
- cluster.transitionToActive(0);
- FileSystem fs = cluster.getFileSystem(0);
- DatanodeProtocolClientSideTranslatorPB spyOnNn0 =
- InternalDataNodeTestUtils.spyOnBposToNN(
- cluster.getDataNodes().get(0), cluster.getNameNode(0));
- DatanodeProtocolClientSideTranslatorPB spyOnNn1 =
- InternalDataNodeTestUtils.spyOnBposToNN(
- cluster.getDataNodes().get(0), cluster.getNameNode(1));
- DFSTestUtil.createFile(fs, new Path("/abc"), 16, (short) 1, 1L);
+ try {
+ cluster.waitActive();
+ cluster.transitionToActive(0);
+ FileSystem fs = cluster.getFileSystem(0);
+ DatanodeProtocolClientSideTranslatorPB spyOnNn0 =
+ InternalDataNodeTestUtils.spyOnBposToNN(
+ cluster.getDataNodes().get(0), cluster.getNameNode(0));
+ DatanodeProtocolClientSideTranslatorPB spyOnNn1 =
+ InternalDataNodeTestUtils.spyOnBposToNN(
+ cluster.getDataNodes().get(0), cluster.getNameNode(1));
+ DFSTestUtil.createFile(fs, new Path("/abc"), 16, (short) 1, 1L);
- // We should get 1 incremental block report on both NNs.
- Mockito.verify(spyOnNn0, timeout(60000).times(1)).blockReceivedAndDeleted(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageReceivedDeletedBlocks[].class));
- Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReceivedAndDeleted(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageReceivedDeletedBlocks[].class));
-
- // We should not receive any more incremental or incremental block reports,
- // since the interval we configured is so long.
- for (int i = 0; i < 3; i++) {
- Thread.sleep(10);
- Mockito.verify(spyOnNn0, times(0)).blockReport(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageBlockReport[].class),
- any());
- Mockito.verify(spyOnNn0, times(1)).blockReceivedAndDeleted(
+ // We should get 1 incremental block report on both NNs.
+ Mockito.verify(spyOnNn0, timeout(60000).times(1)).blockReceivedAndDeleted(
any(DatanodeRegistration.class),
anyString(),
any(StorageReceivedDeletedBlocks[].class));
- Mockito.verify(spyOnNn1, times(0)).blockReport(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageBlockReport[].class),
- any());
- Mockito.verify(spyOnNn1, times(1)).blockReceivedAndDeleted(
+ Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReceivedAndDeleted(
any(DatanodeRegistration.class),
anyString(),
any(StorageReceivedDeletedBlocks[].class));
- }
- // Create a fake block deletion notification on the DataNode.
- // This will be sent with the next incremental block report.
- ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(
- new Block(5678, 512, 1000), BlockStatus.DELETED_BLOCK, null);
- DataNode datanode = cluster.getDataNodes().get(0);
- for (BPServiceActor actor : datanode.getAllBpOs().get(0).getBPServiceActors()) {
- final FsDatasetSpi dataset = datanode.getFSDataset();
- final DatanodeStorage storage;
- try (FsDatasetSpi.FsVolumeReferences volumes = dataset.getFsVolumeReferences()) {
- storage = dataset.getStorage(volumes.get(0).getStorageID());
+ // We should not receive any more incremental or incremental block reports,
+ // since the interval we configured is so long.
+ for (int i = 0; i < 3; i++) {
+ Thread.sleep(10);
+ Mockito.verify(spyOnNn0, times(0)).blockReport(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageBlockReport[].class),
+ any());
+ Mockito.verify(spyOnNn0, times(1)).blockReceivedAndDeleted(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageReceivedDeletedBlocks[].class));
+ Mockito.verify(spyOnNn1, times(0)).blockReport(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageBlockReport[].class),
+ any());
+ Mockito.verify(spyOnNn1, times(1)).blockReceivedAndDeleted(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageReceivedDeletedBlocks[].class));
}
- actor.getIbrManager().addRDBI(rdbi, storage);
- }
- // Manually trigger a block report.
- // Only trigger block report to NN1 when testing triggering block report on specific namenode.
- InetSocketAddress nnAddr = withSpecificNN ? cluster.getNameNode(1).getServiceRpcAddress() : null;
- datanode.triggerBlockReport(
- new BlockReportOptions.Factory().
- setNamenodeAddr(nnAddr).
- setIncremental(incremental).
- build()
- );
+ // Create a fake block deletion notification on the DataNode.
+ // This will be sent with the next incremental block report.
+ ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(
+ new Block(5678, 512, 1000), BlockStatus.DELETED_BLOCK, null);
+ DataNode datanode = cluster.getDataNodes().get(0);
+ for (BPServiceActor actor : datanode.getAllBpOs().get(0).getBPServiceActors()) {
+ final FsDatasetSpi dataset = datanode.getFSDataset();
+ final DatanodeStorage storage;
+ try (FsDatasetSpi.FsVolumeReferences volumes = dataset.getFsVolumeReferences()) {
+ storage = dataset.getStorage(volumes.get(0).getStorageID());
+ }
+ actor.getIbrManager().addRDBI(rdbi, storage);
+ }
- // triggerBlockReport returns before the block report is
- // actually sent. Wait for it to be sent here.
- if (incremental) {
- Mockito.verify(spyOnNn1, timeout(60000).times(2)).
- blockReceivedAndDeleted(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageReceivedDeletedBlocks[].class));
- int nn0IncrBlockReport = withSpecificNN ? 1 : 2;
- Mockito.verify(spyOnNn0, timeout(60000).times(nn0IncrBlockReport)).
- blockReceivedAndDeleted(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageReceivedDeletedBlocks[].class));
- } else {
- Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReport(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageBlockReport[].class),
- any());
- int nn0BlockReport = withSpecificNN ? 0 : 1;
- Mockito.verify(spyOnNn0, timeout(60000).times(nn0BlockReport)).blockReport(
- any(DatanodeRegistration.class),
- anyString(),
- any(StorageBlockReport[].class),
- any());
- }
+ // Manually trigger a block report.
+ // Only trigger block report to NN1 when testing triggering block report on specific namenode.
+ InetSocketAddress nnAddr =
+ withSpecificNN ? cluster.getNameNode(1).getServiceRpcAddress() : null;
+ datanode.triggerBlockReport(
+ new BlockReportOptions.Factory().
+ setNamenodeAddr(nnAddr).
+ setIncremental(incremental).
+ build()
+ );
- cluster.shutdown();
+ // triggerBlockReport returns before the block report is
+ // actually sent. Wait for it to be sent here.
+ if (incremental) {
+ Mockito.verify(spyOnNn1, timeout(60000).times(2)).
+ blockReceivedAndDeleted(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageReceivedDeletedBlocks[].class));
+ int nn0IncrBlockReport = withSpecificNN ? 1 : 2;
+ Mockito.verify(spyOnNn0, timeout(60000).times(nn0IncrBlockReport)).
+ blockReceivedAndDeleted(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageReceivedDeletedBlocks[].class));
+ } else {
+ Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReport(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageBlockReport[].class),
+ any());
+ int nn0BlockReport = withSpecificNN ? 0 : 1;
+ Mockito.verify(spyOnNn0, timeout(60000).times(nn0BlockReport)).blockReport(
+ any(DatanodeRegistration.class),
+ anyString(),
+ any(StorageBlockReport[].class),
+ any());
+ }
+ } finally {
+ cluster.shutdown();
+ }
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 6c00e9690bb91f..53c1e15ff7ea45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -379,47 +379,51 @@ public void testAddRplicaProcessorForAddingReplicaInMap() throws Exception {
cnf.setInt(
DFSConfigKeys.DFS_DATANODE_VOLUMES_REPLICA_ADD_THREADPOOL_SIZE_KEY,
poolSize);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(cnf).numDataNodes(1)
- .storagesPerDatanode(1).build();
- DistributedFileSystem fs = cluster.getFileSystem();
- // Generate data blocks.
- ExecutorService pool = Executors.newFixedThreadPool(10);
- List> futureList = new ArrayList<>();
- for (int i = 0; i < 100; i++) {
- Thread thread = new Thread() {
- @Override
- public void run() {
- for (int j = 0; j < 10; j++) {
- try {
- DFSTestUtil.createFile(fs, new Path("File_" + getName() + j), 10,
- (short) 1, 0);
- } catch (IllegalArgumentException | IOException e) {
- e.printStackTrace();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(cnf)
+ .numDataNodes(1).storagesPerDatanode(1).build()) {
+ DistributedFileSystem fs = cluster.getFileSystem();
+ // Generate data blocks.
+ ExecutorService pool = Executors.newFixedThreadPool(10);
+ List> futureList = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ for (int j = 0; j < 10; j++) {
+ try {
+ DFSTestUtil.createFile(fs, new Path("File_" + getName() + j),
+ 10, (short) 1, 0);
+ } catch (IllegalArgumentException | IOException e) {
+ e.printStackTrace();
+ }
}
}
- }
- };
- thread.setName("FileWriter" + i);
- futureList.add(pool.submit(thread));
- }
- // Wait for data generation
- for (Future f : futureList) {
- f.get();
+ };
+ thread.setName("FileWriter" + i);
+ futureList.add(pool.submit(thread));
+ }
+ // Wait for data generation
+ for (Future f : futureList) {
+ f.get();
+ }
+ pool.shutdown();
+ fs.close();
+ FsDatasetImpl fsDataset = (FsDatasetImpl) cluster.getDataNodes().get(0)
+ .getFSDataset();
+ ReplicaMap volumeMap =
+ new ReplicaMap(fsDataset.acquireDatasetLockManager());
+ RamDiskReplicaTracker ramDiskReplicaMap = RamDiskReplicaTracker
+ .getInstance(conf, fsDataset);
+ FsVolumeImpl vol = (FsVolumeImpl) fsDataset.getFsVolumeReferences().get(0);
+ String bpid = cluster.getNamesystem().getBlockPoolId();
+ // It will create BlockPoolSlice.AddReplicaProcessor task's and lunch in
+ // ForkJoinPool recursively
+ vol.getVolumeMap(bpid, volumeMap, ramDiskReplicaMap);
+ assertTrue(volumeMap.replicas(bpid).size() == 1000,
+ "Failed to add all the replica to map");
+ assertEquals(poolSize, BlockPoolSlice.getAddReplicaForkPoolSize(),
+ "Fork pool should be initialize with configured pool size");
}
- fs.close();
- FsDatasetImpl fsDataset = (FsDatasetImpl) cluster.getDataNodes().get(0)
- .getFSDataset();
- ReplicaMap volumeMap = new ReplicaMap(fsDataset.acquireDatasetLockManager());
- RamDiskReplicaTracker ramDiskReplicaMap = RamDiskReplicaTracker
- .getInstance(conf, fsDataset);
- FsVolumeImpl vol = (FsVolumeImpl) fsDataset.getFsVolumeReferences().get(0);
- String bpid = cluster.getNamesystem().getBlockPoolId();
- // It will create BlockPoolSlice.AddReplicaProcessor task's and lunch in
- // ForkJoinPool recursively
- vol.getVolumeMap(bpid, volumeMap, ramDiskReplicaMap);
- assertTrue(volumeMap.replicas(bpid).size() == 1000, "Failed to add all the replica to map");
- assertEquals(poolSize, BlockPoolSlice.getAddReplicaForkPoolSize(),
- "Fork pool should be initialize with configured pool size");
}
@Test
@@ -648,77 +652,78 @@ public void testExcludeSlowDiskWhenChoosingVolume() throws Exception {
}
}
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.hosts(hostnames)
.numDataNodes(NUM_DATANODES)
.storagesPerDatanode(STORAGES_PER_DATANODE)
- .storageCapacities(capacities).build();
- cluster.waitActive();
- FileSystem fs = cluster.getFileSystem();
-
- // Create file for each datanode.
- ArrayList dataNodes = cluster.getDataNodes();
- DataNode dn0 = dataNodes.get(0);
- DataNode dn1 = dataNodes.get(1);
- DataNode dn2 = dataNodes.get(2);
-
- // Mock the first disk of each datanode is a slowest disk.
- String slowDisk0OnDn0 = dn0.getFSDataset().getFsVolumeReferences().getReference(0)
- .getVolume().getBaseURI().getPath();
- String slowDisk0OnDn1 = dn1.getFSDataset().getFsVolumeReferences().getReference(0)
- .getVolume().getBaseURI().getPath();
- String slowDisk0OnDn2 = dn2.getFSDataset().getFsVolumeReferences().getReference(0)
- .getVolume().getBaseURI().getPath();
-
- String slowDisk1OnDn0 = dn0.getFSDataset().getFsVolumeReferences().getReference(1)
- .getVolume().getBaseURI().getPath();
- String slowDisk1OnDn1 = dn1.getFSDataset().getFsVolumeReferences().getReference(1)
- .getVolume().getBaseURI().getPath();
- String slowDisk1OnDn2 = dn2.getFSDataset().getFsVolumeReferences().getReference(1)
- .getVolume().getBaseURI().getPath();
-
- dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn0, ImmutableMap.of(
- SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
- SlowDiskReports.DiskOp.METADATA, 2.0));
- dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn1, ImmutableMap.of(
- SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
- SlowDiskReports.DiskOp.METADATA, 2.0));
- dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn2, ImmutableMap.of(
- SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
- SlowDiskReports.DiskOp.METADATA, 2.0));
-
- dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn0, ImmutableMap.of(
- SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
- SlowDiskReports.DiskOp.METADATA, 1.0));
- dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn1, ImmutableMap.of(
- SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
- SlowDiskReports.DiskOp.METADATA, 1.0));
- dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn2, ImmutableMap.of(
- SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
- SlowDiskReports.DiskOp.METADATA, 1.0));
-
- // Wait until the data on the slow disk is collected successfully.
- GenericTestUtils.waitFor(new Supplier() {
- @Override public Boolean get() {
- return dn0.getDiskMetrics().getSlowDisksToExclude().size() == 1 &&
- dn1.getDiskMetrics().getSlowDisksToExclude().size() == 1 &&
- dn2.getDiskMetrics().getSlowDisksToExclude().size() == 1;
- }
- }, 1000, 5000);
-
- // Create a file with 3 replica.
- DFSTestUtil.createFile(fs, new Path("/file0"), false, BUFFER_LENGTH, 1000,
- DEFAULT_BLOCK_SIZE, (short) 3, 0, false, null);
-
- // Asserts that the number of blocks created on a slow disk is 0.
- assertEquals(0,
- dn0.getVolumeReport().stream().filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn0))
- .collect(Collectors.toList()).get(0).getNumBlocks());
- assertEquals(0,
- dn1.getVolumeReport().stream().filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn1))
- .collect(Collectors.toList()).get(0).getNumBlocks());
- assertEquals(0,
- dn2.getVolumeReport().stream().filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn2))
- .collect(Collectors.toList()).get(0).getNumBlocks());
+ .storageCapacities(capacities).build()) {
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+
+ // Create file for each datanode.
+ ArrayList dataNodes = cluster.getDataNodes();
+ DataNode dn0 = dataNodes.get(0);
+ DataNode dn1 = dataNodes.get(1);
+ DataNode dn2 = dataNodes.get(2);
+
+ // Mock the first disk of each datanode is a slowest disk.
+ String slowDisk0OnDn0 = dn0.getFSDataset().getFsVolumeReferences()
+ .getReference(0).getVolume().getBaseURI().getPath();
+ String slowDisk0OnDn1 = dn1.getFSDataset().getFsVolumeReferences()
+ .getReference(0).getVolume().getBaseURI().getPath();
+ String slowDisk0OnDn2 = dn2.getFSDataset().getFsVolumeReferences()
+ .getReference(0).getVolume().getBaseURI().getPath();
+
+ String slowDisk1OnDn0 = dn0.getFSDataset().getFsVolumeReferences()
+ .getReference(1).getVolume().getBaseURI().getPath();
+ String slowDisk1OnDn1 = dn1.getFSDataset().getFsVolumeReferences()
+ .getReference(1).getVolume().getBaseURI().getPath();
+ String slowDisk1OnDn2 = dn2.getFSDataset().getFsVolumeReferences()
+ .getReference(1).getVolume().getBaseURI().getPath();
+
+ dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn0, ImmutableMap.of(
+ SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
+ SlowDiskReports.DiskOp.METADATA, 2.0));
+ dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn1, ImmutableMap.of(
+ SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
+ SlowDiskReports.DiskOp.METADATA, 2.0));
+ dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn2, ImmutableMap.of(
+ SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5,
+ SlowDiskReports.DiskOp.METADATA, 2.0));
+
+ dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn0, ImmutableMap.of(
+ SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
+ SlowDiskReports.DiskOp.METADATA, 1.0));
+ dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn1, ImmutableMap.of(
+ SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
+ SlowDiskReports.DiskOp.METADATA, 1.0));
+ dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn2, ImmutableMap.of(
+ SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0,
+ SlowDiskReports.DiskOp.METADATA, 1.0));
+
+ // Wait until the data on the slow disk is collected successfully.
+ GenericTestUtils.waitFor(new Supplier() {
+ @Override public Boolean get() {
+ return dn0.getDiskMetrics().getSlowDisksToExclude().size() == 1 &&
+ dn1.getDiskMetrics().getSlowDisksToExclude().size() == 1 &&
+ dn2.getDiskMetrics().getSlowDisksToExclude().size() == 1;
+ }
+ }, 1000, 5000);
+
+ // Create a file with 3 replica.
+ DFSTestUtil.createFile(fs, new Path("/file0"), false, BUFFER_LENGTH, 1000,
+ DEFAULT_BLOCK_SIZE, (short) 3, 0, false, null);
+
+ // Asserts that the number of blocks created on a slow disk is 0.
+ assertEquals(0, dn0.getVolumeReport().stream()
+ .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn0))
+ .collect(Collectors.toList()).get(0).getNumBlocks());
+ assertEquals(0, dn1.getVolumeReport().stream()
+ .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn1))
+ .collect(Collectors.toList()).get(0).getNumBlocks());
+ assertEquals(0, dn2.getVolumeReport().stream()
+ .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn2))
+ .collect(Collectors.toList()).get(0).getNumBlocks());
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java
index 0c6cb6751782d0..27e5c9a2083ee8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java
@@ -605,42 +605,43 @@ public void testScannerWithProvidedVolumes() throws Exception {
public void testProvidedReplicaWithPathHandle() throws Exception {
Configuration conf = new Configuration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- cluster.waitActive();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+ cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
+ DistributedFileSystem fs = cluster.getFileSystem();
- // generate random data
- int chunkSize = 512;
- Random r = new Random(12345L);
- byte[] data = new byte[chunkSize];
- r.nextBytes(data);
+ // generate random data
+ int chunkSize = 512;
+ Random r = new Random(12345L);
+ byte[] data = new byte[chunkSize];
+ r.nextBytes(data);
- Path file = new Path("/testfile");
- try (FSDataOutputStream fout = fs.create(file)) {
- fout.write(data);
- }
+ Path file = new Path("/testfile");
+ try (FSDataOutputStream fout = fs.create(file)) {
+ fout.write(data);
+ }
- PathHandle pathHandle = fs.getPathHandle(fs.getFileStatus(file),
- Options.HandleOpt.changed(true), Options.HandleOpt.moved(true));
- FinalizedProvidedReplica replica = new FinalizedProvidedReplica(0,
- file.toUri(), 0, chunkSize, 0, pathHandle, null, conf, fs);
- byte[] content = new byte[chunkSize];
- IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize);
- assertArrayEquals(data, content);
-
- fs.rename(file, new Path("/testfile.1"));
- // read should continue succeeding after the rename operation
- IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize);
- assertArrayEquals(data, content);
-
- replica.setPathHandle(null);
- try {
- // expected to fail as URI of the provided replica is no longer valid.
- replica.getDataInputStream(0);
- fail("Expected an exception");
- } catch (IOException e) {
- LOG.info("Expected exception " + e);
+ PathHandle pathHandle = fs.getPathHandle(fs.getFileStatus(file),
+ Options.HandleOpt.changed(true), Options.HandleOpt.moved(true));
+ FinalizedProvidedReplica replica = new FinalizedProvidedReplica(0,
+ file.toUri(), 0, chunkSize, 0, pathHandle, null, conf, fs);
+ byte[] content = new byte[chunkSize];
+ IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize);
+ assertArrayEquals(data, content);
+
+ fs.rename(file, new Path("/testfile.1"));
+ // read should continue succeeding after the rename operation
+ IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize);
+ assertArrayEquals(data, content);
+
+ replica.setPathHandle(null);
+ try {
+ // expected to fail as URI of the provided replica is no longer valid.
+ replica.getDataInputStream(0);
+ fail("Expected an exception");
+ } catch (IOException e) {
+ LOG.info("Expected exception " + e);
+ }
}
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index cf78c187ae2917..1765d8cc5235ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -572,40 +572,41 @@ public void testReplicaMapAfterDatanodeRestart() throws Exception {
@Test
public void testRecoverInconsistentRbw() throws IOException {
Configuration conf = new HdfsConfiguration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf,
- new File(GenericTestUtils.getRandomizedTempPath())).build();
- cluster.waitActive();
- DataNode dn = cluster.getDataNodes().get(0);
- FsDatasetImpl fsDataset = (FsDatasetImpl)DataNodeTestUtils.getFSDataset(dn);
-
- // set up replicasMap
- String bpid = cluster.getNamesystem().getBlockPoolId();
- ExtendedBlock[] blocks = setup(bpid, cluster.getFsDatasetTestUtils(dn));
-
- ReplicaBeingWritten rbw = (ReplicaBeingWritten)fsDataset.
- getReplicaInfo(bpid, blocks[RBW].getBlockId());
- long bytesOnDisk = rbw.getBytesOnDisk();
- // simulate an inconsistent replica length update by reducing in-memory
- // value of on disk length
- rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null);
- fsDataset.recoverRbw(blocks[RBW], blocks[RBW].getGenerationStamp(), 0L,
- rbw.getNumBytes());
- // after the recovery, on disk length should equal acknowledged length.
- assertTrue(rbw.getBytesOnDisk() == rbw.getBytesAcked());
-
- // reduce on disk length again; this time actually truncate the file to
- // simulate the data not being present
- rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null);
- try (RandomAccessFile blockRAF = rbw.getFileIoProvider().
- getRandomAccessFile(rbw.getVolume(), rbw.getBlockFile(), "rw")) {
- // truncate blockFile
- blockRAF.setLength(bytesOnDisk - 1);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf,
+ new File(GenericTestUtils.getRandomizedTempPath())).build()) {
+ cluster.waitActive();
+ DataNode dn = cluster.getDataNodes().get(0);
+ FsDatasetImpl fsDataset = (FsDatasetImpl)DataNodeTestUtils.getFSDataset(dn);
+
+ // set up replicasMap
+ String bpid = cluster.getNamesystem().getBlockPoolId();
+ ExtendedBlock[] blocks = setup(bpid, cluster.getFsDatasetTestUtils(dn));
+
+ ReplicaBeingWritten rbw = (ReplicaBeingWritten)fsDataset.
+ getReplicaInfo(bpid, blocks[RBW].getBlockId());
+ long bytesOnDisk = rbw.getBytesOnDisk();
+ // simulate an inconsistent replica length update by reducing in-memory
+ // value of on disk length
+ rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null);
fsDataset.recoverRbw(blocks[RBW], blocks[RBW].getGenerationStamp(), 0L,
rbw.getNumBytes());
- fail("recovery should have failed");
- } catch (ReplicaNotFoundException rnfe) {
- GenericTestUtils.assertExceptionContains("Found fewer bytesOnDisk than " +
- "bytesAcked for replica", rnfe);
+ // after the recovery, on disk length should equal acknowledged length.
+ assertTrue(rbw.getBytesOnDisk() == rbw.getBytesAcked());
+
+ // reduce on disk length again; this time actually truncate the file to
+ // simulate the data not being present
+ rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null);
+ try (RandomAccessFile blockRAF = rbw.getFileIoProvider().
+ getRandomAccessFile(rbw.getVolume(), rbw.getBlockFile(), "rw")) {
+ // truncate blockFile
+ blockRAF.setLength(bytesOnDisk - 1);
+ fsDataset.recoverRbw(blocks[RBW], blocks[RBW].getGenerationStamp(), 0L,
+ rbw.getNumBytes());
+ fail("recovery should have failed");
+ } catch (ReplicaNotFoundException rnfe) {
+ GenericTestUtils.assertExceptionContains("Found fewer bytesOnDisk than " +
+ "bytesAcked for replica", rnfe);
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index 8e4e3dc9ae67b1..d5aae7fd8e7d71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -1451,47 +1451,48 @@ public void testMoverMetrics() throws Exception {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, blockSize);
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(2)
.storageTypes(
new StorageType[][] {{StorageType.DISK, StorageType.DISK},
{StorageType.ARCHIVE, StorageType.ARCHIVE}})
- .build();
+ .build()) {
+
+ cluster.waitActive();
+ final DistributedFileSystem fs = cluster.getFileSystem();
+
+ final String file = "/testMaxIterationTime.dat";
+ final Path path = new Path(file);
+ short repFactor = 1;
+ int seed = 0xFAFAFA;
+ // write to DISK
+ DFSTestUtil.createFile(fs, path, 4L * blockSize, repFactor, seed);
- cluster.waitActive();
- final DistributedFileSystem fs = cluster.getFileSystem();
-
- final String file = "/testMaxIterationTime.dat";
- final Path path = new Path(file);
- short repFactor = 1;
- int seed = 0xFAFAFA;
- // write to DISK
- DFSTestUtil.createFile(fs, path, 4L * blockSize, repFactor, seed);
-
- // move to ARCHIVE
- fs.setStoragePolicy(new Path(file), "COLD");
-
- Map> nnWithPath = new HashMap<>();
- List paths = new ArrayList<>();
- paths.add(path);
- nnWithPath
- .put(DFSUtil.getInternalNsRpcUris(conf).iterator().next(), paths);
-
- Mover.run(nnWithPath, conf);
-
- final String moverMetricsName = "Mover-"
- + cluster.getNameNode(0).getNamesystem().getBlockPoolId();
- MetricsSource moverMetrics =
- DefaultMetricsSystem.instance().getSource(moverMetricsName);
- assertNotNull(moverMetrics);
-
- MetricsRecordBuilder rb = MetricsAsserts.getMetrics(moverMetricsName);
- // Check metrics
- assertEquals(4, MetricsAsserts.getLongCounter("BlocksScheduled", rb));
- assertEquals(1, MetricsAsserts.getLongCounter("FilesProcessed", rb));
- assertEquals(41943040, MetricsAsserts.getLongGauge("BytesMoved", rb));
- assertEquals(4, MetricsAsserts.getLongGauge("BlocksMoved", rb));
- assertEquals(0, MetricsAsserts.getLongGauge("BlocksFailed", rb));
+ // move to ARCHIVE
+ fs.setStoragePolicy(new Path(file), "COLD");
+
+ Map> nnWithPath = new HashMap<>();
+ List paths = new ArrayList<>();
+ paths.add(path);
+ nnWithPath
+ .put(DFSUtil.getInternalNsRpcUris(conf).iterator().next(), paths);
+
+ Mover.run(nnWithPath, conf);
+
+ final String moverMetricsName = "Mover-"
+ + cluster.getNameNode(0).getNamesystem().getBlockPoolId();
+ MetricsSource moverMetrics =
+ DefaultMetricsSystem.instance().getSource(moverMetricsName);
+ assertNotNull(moverMetrics);
+
+ MetricsRecordBuilder rb = MetricsAsserts.getMetrics(moverMetricsName);
+ // Check metrics
+ assertEquals(4, MetricsAsserts.getLongCounter("BlocksScheduled", rb));
+ assertEquals(1, MetricsAsserts.getLongCounter("FilesProcessed", rb));
+ assertEquals(41943040, MetricsAsserts.getLongGauge("BytesMoved", rb));
+ assertEquals(4, MetricsAsserts.getLongGauge("BlocksMoved", rb));
+ assertEquals(0, MetricsAsserts.getLongGauge("BlocksFailed", rb));
+ }
}
private void createFileWithFavoredDatanodes(final Configuration conf,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
index ea4ed0fc9311e0..8da52f0629be70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
@@ -1663,60 +1663,63 @@ DistributedFileSystem getDFS(MiniDFSCluster cluster, int nnIdx)
public void testExpiryTimeConsistency() throws Exception {
conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
- MiniDFSCluster dfsCluster =
+ try (MiniDFSCluster dfsCluster =
new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
.nnTopology(MiniDFSNNTopology.simpleHATopology())
- .build();
- dfsCluster.transitionToActive(0);
-
- DistributedFileSystem fs = getDFS(dfsCluster, 0);
- final NameNode ann = dfsCluster.getNameNode(0);
-
- final Path filename = new Path("/file");
- final short replication = (short) 3;
- DFSTestUtil.createFile(fs, filename, 1, replication, 0x0BAC);
- fs.addCachePool(new CachePoolInfo("pool"));
- long id = fs.addCacheDirective(
- new CacheDirectiveInfo.Builder().setPool("pool").setPath(filename)
- .setExpiration(CacheDirectiveInfo.Expiration.newRelative(86400000))
- .setReplication(replication).build());
- fs.modifyCacheDirective(new CacheDirectiveInfo.Builder()
- .setId(id)
- .setExpiration(CacheDirectiveInfo.Expiration.newRelative(172800000))
- .build());
- final NameNode sbn = dfsCluster.getNameNode(1);
- final CacheManager annCachemanager = ann.getNamesystem().getCacheManager();
- final CacheManager sbnCachemanager = sbn.getNamesystem().getCacheManager();
- HATestUtil.waitForStandbyToCatchUp(ann, sbn);
- GenericTestUtils.waitFor(() -> {
- boolean isConsistence = false;
- ann.getNamesystem().readLock(RwLockMode.FS);
- try {
- sbn.getNamesystem().readLock(RwLockMode.FS);
+ .build()) {
+ dfsCluster.transitionToActive(0);
+
+ DistributedFileSystem fs = getDFS(dfsCluster, 0);
+ final NameNode ann = dfsCluster.getNameNode(0);
+
+ final Path filename = new Path("/file");
+ final short replication = (short) 3;
+ DFSTestUtil.createFile(fs, filename, 1, replication, 0x0BAC);
+ fs.addCachePool(new CachePoolInfo("pool"));
+ long id = fs.addCacheDirective(
+ new CacheDirectiveInfo.Builder().setPool("pool").setPath(filename)
+ .setExpiration(CacheDirectiveInfo.Expiration.newRelative(86400000))
+ .setReplication(replication).build());
+ fs.modifyCacheDirective(new CacheDirectiveInfo.Builder()
+ .setId(id)
+ .setExpiration(CacheDirectiveInfo.Expiration.newRelative(172800000))
+ .build());
+ final NameNode sbn = dfsCluster.getNameNode(1);
+ final CacheManager annCachemanager = ann.getNamesystem().getCacheManager();
+ final CacheManager sbnCachemanager = sbn.getNamesystem().getCacheManager();
+ HATestUtil.waitForStandbyToCatchUp(ann, sbn);
+ GenericTestUtils.waitFor(() -> {
+ boolean isConsistence = false;
+ ann.getNamesystem().readLock(RwLockMode.FS);
try {
- Iterator annDirectivesIt = annCachemanager.
- getCacheDirectives().iterator();
- Iterator sbnDirectivesIt = sbnCachemanager.
- getCacheDirectives().iterator();
- if (annDirectivesIt.hasNext() && sbnDirectivesIt.hasNext()) {
- CacheDirective annDirective = annDirectivesIt.next();
- CacheDirective sbnDirective = sbnDirectivesIt.next();
- if (annDirective.getExpiryTimeString().
- equals(sbnDirective.getExpiryTimeString())) {
- isConsistence = true;
+ sbn.getNamesystem().readLock(RwLockMode.FS);
+ try {
+ Iterator annDirectivesIt = annCachemanager.
+ getCacheDirectives().iterator();
+ Iterator sbnDirectivesIt = sbnCachemanager.
+ getCacheDirectives().iterator();
+ if (annDirectivesIt.hasNext() && sbnDirectivesIt.hasNext()) {
+ CacheDirective annDirective = annDirectivesIt.next();
+ CacheDirective sbnDirective = sbnDirectivesIt.next();
+ if (annDirective.getExpiryTimeString().
+ equals(sbnDirective.getExpiryTimeString())) {
+ isConsistence = true;
+ }
}
+ } finally {
+ sbn.getNamesystem().readUnlock(RwLockMode.FS,
+ "expiryTimeConsistency");
}
} finally {
- sbn.getNamesystem().readUnlock(RwLockMode.FS, "expiryTimeConsistency");
+ ann.getNamesystem().readUnlock(RwLockMode.FS,
+ "expiryTimeConsistency");
}
- } finally {
- ann.getNamesystem().readUnlock(RwLockMode.FS, "expiryTimeConsistency");
- }
- if (!isConsistence) {
- LOG.info("testEexpiryTimeConsistency:"
- + "ANN CacheDirective Status is inconsistent with SBN");
- }
- return isConsistence;
- }, 500, 120000);
+ if (!isConsistence) {
+ LOG.info("testEexpiryTimeConsistency:"
+ + "ANN CacheDirective Status is inconsistent with SBN");
+ }
+ return isConsistence;
+ }, 500, 120000);
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java
index 64df2e6f1147f1..ec6bb6cdfc0312 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -52,6 +53,14 @@ public static void setUp() throws IOException {
dfs = cluster.getFileSystem();
}
+ @AfterAll
+ public static void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
@Test
public void testQuotaUsageWhenRenameWithSameStoragePolicy() throws Exception {
final int fileLen = 1024;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 8967001b7b9bea..31510a3efc7c06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -1168,59 +1168,60 @@ private void ensureSubSectionsAlignWithParent(ArrayList subSec,
@Test
public void testUpdateBlocksMapAndNameCacheAsync() throws IOException {
Configuration conf = new Configuration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
- FSDirectory fsdir = cluster.getNameNode().namesystem.getFSDirectory();
- File workingDir = GenericTestUtils.getTestDir();
-
- File preRestartTree = new File(workingDir, "preRestartTree");
- File postRestartTree = new File(workingDir, "postRestartTree");
-
- Path baseDir = new Path("/user/foo");
- fs.mkdirs(baseDir);
- fs.allowSnapshot(baseDir);
- for (int i = 0; i < 5; i++) {
- Path dir = new Path(baseDir, Integer.toString(i));
- fs.mkdirs(dir);
- for (int j = 0; j < 5; j++) {
- Path file = new Path(dir, Integer.toString(j));
- FSDataOutputStream os = fs.create(file);
- os.write((byte) j);
- os.close();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) {
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSDirectory fsdir = cluster.getNameNode().namesystem.getFSDirectory();
+ File workingDir = GenericTestUtils.getTestDir();
+
+ File preRestartTree = new File(workingDir, "preRestartTree");
+ File postRestartTree = new File(workingDir, "postRestartTree");
+
+ Path baseDir = new Path("/user/foo");
+ fs.mkdirs(baseDir);
+ fs.allowSnapshot(baseDir);
+ for (int i = 0; i < 5; i++) {
+ Path dir = new Path(baseDir, Integer.toString(i));
+ fs.mkdirs(dir);
+ for (int j = 0; j < 5; j++) {
+ Path file = new Path(dir, Integer.toString(j));
+ FSDataOutputStream os = fs.create(file);
+ os.write((byte) j);
+ os.close();
+ }
+ fs.createSnapshot(baseDir, "snap_"+i);
+ fs.rename(new Path(dir, "0"), new Path(dir, "renamed"));
}
- fs.createSnapshot(baseDir, "snap_"+i);
- fs.rename(new Path(dir, "0"), new Path(dir, "renamed"));
- }
- SnapshotTestHelper.dumpTree2File(fsdir, preRestartTree);
-
- // checkpoint
- fs.setSafeMode(SafeModeAction.ENTER);
- fs.saveNamespace();
- fs.setSafeMode(SafeModeAction.LEAVE);
+ SnapshotTestHelper.dumpTree2File(fsdir, preRestartTree);
- cluster.restartNameNode();
- cluster.waitActive();
- fs = cluster.getFileSystem();
- fsdir = cluster.getNameNode().namesystem.getFSDirectory();
+ // checkpoint
+ fs.setSafeMode(SafeModeAction.ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.LEAVE);
- // Ensure all the files created above exist, and blocks is correct.
- for (int i = 0; i < 5; i++) {
- Path dir = new Path(baseDir, Integer.toString(i));
- assertTrue(fs.getFileStatus(dir).isDirectory());
- for (int j = 0; j < 5; j++) {
- Path file = new Path(dir, Integer.toString(j));
- if (j == 0) {
- file = new Path(dir, "renamed");
+ cluster.restartNameNode();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ fsdir = cluster.getNameNode().namesystem.getFSDirectory();
+
+ // Ensure all the files created above exist, and blocks is correct.
+ for (int i = 0; i < 5; i++) {
+ Path dir = new Path(baseDir, Integer.toString(i));
+ assertTrue(fs.getFileStatus(dir).isDirectory());
+ for (int j = 0; j < 5; j++) {
+ Path file = new Path(dir, Integer.toString(j));
+ if (j == 0) {
+ file = new Path(dir, "renamed");
+ }
+ FSDataInputStream in = fs.open(file);
+ int n = in.readByte();
+ assertEquals(j, n);
+ in.close();
}
- FSDataInputStream in = fs.open(file);
- int n = in.readByte();
- assertEquals(j, n);
- in.close();
}
+ SnapshotTestHelper.dumpTree2File(fsdir, postRestartTree);
+ SnapshotTestHelper.compareDumpedTreeInFile(
+ preRestartTree, postRestartTree, true);
}
- SnapshotTestHelper.dumpTree2File(fsdir, postRestartTree);
- SnapshotTestHelper.compareDumpedTreeInFile(
- preRestartTree, postRestartTree, true);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
index 7c203208ee4c63..70dffbb0c7dffe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
@@ -105,7 +105,7 @@ private static String getPreferredLocation(DistributedFileSystem fs,
static final int ITERATIONS_TO_USE = 20;
@Test
- @Timeout(30000)
+ @Timeout(30)
public void testNamenodeRpcClientIpProxyWithFailBack() throws Exception {
// Make 3 nodes & racks so that we have a decent shot of detecting when
// our change overrides the random choice of datanode.
@@ -165,7 +165,7 @@ public void testNamenodeRpcClientIpProxyWithFailBack() throws Exception {
}
@Test
- @Timeout(30000)
+ @Timeout(30)
public void testObserverHandleAddBlock() throws Exception {
String baseDir = GenericTestUtils.getRandomizedTempPath();
Configuration conf = new HdfsConfiguration();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java
index 87a582c02598df..3a9119d350dfd0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java
@@ -52,57 +52,58 @@ public void testPrintTopologyTextFormat() throws IOException {
}
}
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(dataNodesNum)
.racks(rackList.toArray(new String[rackList.size()]))
- .build();
- cluster.waitActive();
-
- // get http uri
- String httpUri = cluster.getHttpUri(0);
-
- // send http request
- URL url = new URL(httpUri + "/topology");
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.setReadTimeout(20000);
- conn.setConnectTimeout(20000);
- conn.connect();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
- StringBuilder sb =
- new StringBuilder("-- Network Topology -- \n");
- sb.append(out);
- sb.append("\n-- Network Topology -- ");
- String topology = sb.toString();
-
- // assert rack info
- assertTrue(topology.contains("/rack0"));
- assertTrue(topology.contains("/rack1"));
- assertTrue(topology.contains("/rack2"));
- assertTrue(topology.contains("/rack3"));
- assertTrue(topology.contains("/rack4"));
-
- // assert node number
- assertEquals(topology.split("127.0.0.1").length - 1, dataNodesNum);
+ .build()) {
+ cluster.waitActive();
+
+ // get http uri
+ String httpUri = cluster.getHttpUri(0);
+
+ // send http request
+ URL url = new URL(httpUri + "/topology");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(20000);
+ conn.setConnectTimeout(20000);
+ conn.connect();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+ StringBuilder sb =
+ new StringBuilder("-- Network Topology -- \n");
+ sb.append(out);
+ sb.append("\n-- Network Topology -- ");
+ String topology = sb.toString();
+
+ // assert rack info
+ assertTrue(topology.contains("/rack0"));
+ assertTrue(topology.contains("/rack1"));
+ assertTrue(topology.contains("/rack2"));
+ assertTrue(topology.contains("/rack3"));
+ assertTrue(topology.contains("/rack4"));
+
+ // assert node number
+ assertEquals(topology.split("127.0.0.1").length - 1, dataNodesNum);
+ }
}
@Test
public void testPrintTopologyJsonFormat() throws IOException {
- StaticMapping.resetMap();
- Configuration conf = new HdfsConfiguration();
- int dataNodesNum = 0;
- final ArrayList rackList = new ArrayList();
- for (int i = 0; i < 5; i++) {
- for (int j = 0; j < 2; j++) {
- rackList.add("/rack" + i);
- dataNodesNum++;
- }
+ StaticMapping.resetMap();
+ Configuration conf = new HdfsConfiguration();
+ int dataNodesNum = 0;
+ final ArrayList rackList = new ArrayList();
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 2; j++) {
+ rackList.add("/rack" + i);
+ dataNodesNum++;
}
+ }
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(dataNodesNum)
- .racks(rackList.toArray(new String[rackList.size()]))
- .build();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(dataNodesNum)
+ .racks(rackList.toArray(new String[rackList.size()]))
+ .build()) {
cluster.waitActive();
// get http uri
@@ -128,74 +129,77 @@ public void testPrintTopologyJsonFormat() throws IOException {
// assert node number
Iterator elements = racks.elements();
int dataNodesCount = 0;
- while(elements.hasNext()){
- JsonNode rack = elements.next();
- Iterator> fields = rack.fields();
- while (fields.hasNext()) {
- dataNodesCount += fields.next().getValue().size();
- }
+ while (elements.hasNext()) {
+ JsonNode rack = elements.next();
+ Iterator> fields = rack.fields();
+ while (fields.hasNext()) {
+ dataNodesCount += fields.next().getValue().size();
+ }
}
assertEquals(dataNodesCount, dataNodesNum);
+ }
}
@Test
public void testPrintTopologyNoDatanodesTextFormat() throws IOException {
StaticMapping.resetMap();
Configuration conf = new HdfsConfiguration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(0)
- .build();
- cluster.waitActive();
-
- // get http uri
- String httpUri = cluster.getHttpUri(0);
-
- // send http request
- URL url = new URL(httpUri + "/topology");
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.setReadTimeout(20000);
- conn.setConnectTimeout(20000);
- conn.connect();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
- StringBuilder sb =
- new StringBuilder("-- Network Topology -- \n");
- sb.append(out);
- sb.append("\n-- Network Topology -- ");
- String topology = sb.toString();
-
- // assert node number
- assertTrue(topology.contains("No DataNodes"));
+ .build()) {
+ cluster.waitActive();
+
+ // get http uri
+ String httpUri = cluster.getHttpUri(0);
+
+ // send http request
+ URL url = new URL(httpUri + "/topology");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(20000);
+ conn.setConnectTimeout(20000);
+ conn.connect();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+ StringBuilder sb =
+ new StringBuilder("-- Network Topology -- \n");
+ sb.append(out);
+ sb.append("\n-- Network Topology -- ");
+ String topology = sb.toString();
+
+ // assert node number
+ assertTrue(topology.contains("No DataNodes"));
+ }
}
- @Test
- public void testPrintTopologyNoDatanodesJsonFormat() throws IOException {
- StaticMapping.resetMap();
- Configuration conf = new HdfsConfiguration();
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(0)
- .build();
- cluster.waitActive();
-
- // get http uri
- String httpUri = cluster.getHttpUri(0);
-
- // send http request
- URL url = new URL(httpUri + "/topology");
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.setReadTimeout(20000);
- conn.setConnectTimeout(20000);
- conn.setRequestProperty("Accept", "application/json");
- conn.connect();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
- StringBuilder sb =
- new StringBuilder("-- Network Topology -- \n");
- sb.append(out);
- sb.append("\n-- Network Topology -- ");
- String topology = sb.toString();
-
- // assert node number
- assertTrue(topology.contains("No DataNodes"));
+ @Test
+ public void testPrintTopologyNoDatanodesJsonFormat() throws IOException {
+ StaticMapping.resetMap();
+ Configuration conf = new HdfsConfiguration();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0)
+ .build()) {
+ cluster.waitActive();
+
+ // get http uri
+ String httpUri = cluster.getHttpUri(0);
+
+ // send http request
+ URL url = new URL(httpUri + "/topology");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setReadTimeout(20000);
+ conn.setConnectTimeout(20000);
+ conn.setRequestProperty("Accept", "application/json");
+ conn.connect();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ IOUtils.copyBytes(conn.getInputStream(), out, 4096, true);
+ StringBuilder sb =
+ new StringBuilder("-- Network Topology -- \n");
+ sb.append(out);
+ sb.append("\n-- Network Topology -- ");
+ String topology = sb.toString();
+
+ // assert node number
+ assertTrue(topology.contains("No DataNodes"));
}
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
index eb52ff0a30c72b..823bcfaac76ddb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
@@ -318,7 +318,7 @@ public void testMultipleSatisfyStoragePolicy() throws Exception {
* @throws Exception
*/
@Test
- @Timeout(value = 300000)
+ @Timeout(value = 300)
public void testDropSPS() throws Exception {
try {
clusterSetUp();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java
index c76c759cc24e3a..9fe2696c227ecb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java
@@ -222,34 +222,38 @@ public void testReconfigureProtectedPaths() throws Throwable {
MiniDFSCluster cluster = setupTestCase(conf, protectedPaths,
unprotectedPaths);
- SortedSet protectedPathsNew = new TreeSet<>(
- FSDirectory.normalizePaths(Arrays.asList("/aa", "/bb", "/cc"),
- FS_PROTECTED_DIRECTORIES));
+ try {
+ SortedSet protectedPathsNew = new TreeSet<>(
+ FSDirectory.normalizePaths(Arrays.asList("/aa", "/bb", "/cc"),
+ FS_PROTECTED_DIRECTORIES));
- String protectedPathsStrNew = "/aa,/bb,/cc";
+ String protectedPathsStrNew = "/aa,/bb,/cc";
- NameNode nn = cluster.getNameNode();
+ NameNode nn = cluster.getNameNode();
- // change properties
- nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, protectedPathsStrNew);
+ // change properties
+ nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, protectedPathsStrNew);
- FSDirectory fsDirectory = nn.getNamesystem().getFSDirectory();
- // verify change
- assertEquals(protectedPathsNew, fsDirectory.getProtectedDirectories(),
- String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
+ FSDirectory fsDirectory = nn.getNamesystem().getFSDirectory();
+ // verify change
+ assertEquals(protectedPathsNew, fsDirectory.getProtectedDirectories(),
+ String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
- assertEquals(protectedPathsStrNew, nn.getConf().get(FS_PROTECTED_DIRECTORIES),
- String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
+ assertEquals(protectedPathsStrNew, nn.getConf().get(FS_PROTECTED_DIRECTORIES),
+ String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
- // revert to default
- nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, null);
+ // revert to default
+ nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, null);
- // verify default
- assertEquals(new TreeSet(), fsDirectory.getProtectedDirectories(),
- String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
+ // verify default
+ assertEquals(new TreeSet(), fsDirectory.getProtectedDirectories(),
+ String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
- assertEquals(null, nn.getConf().get(FS_PROTECTED_DIRECTORIES),
- String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
+ assertEquals(null, nn.getConf().get(FS_PROTECTED_DIRECTORIES),
+ String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES));
+ } finally {
+ cluster.shutdown();
+ }
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java
index 76a83f6a27595c..45f7675a8d254d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -60,6 +61,14 @@ public void setup() throws Exception {
cluster.waitActive();
}
+ @AfterEach
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
@Test
public void testAliasmapBootstrap() throws Exception {
InMemoryLevelDBAliasMapServer aliasMapServer =
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java
index 39b6bf8aa32f3f..405c8ccf25e536 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java
@@ -192,36 +192,38 @@ public void testGetNameNodeState() throws IOException {
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, Integer.MAX_VALUE);
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
- MiniDFSNNTopology.simpleHATopology(3)).numDataNodes(1).build();
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(
+ MiniDFSNNTopology.simpleHATopology(3)).numDataNodes(1).build()) {
- cluster.waitActive();
+ cluster.waitActive();
- NameNode nn0 = cluster.getNameNode(0);
- NameNode nn1 = cluster.getNameNode(1);
- NameNode nn2 = cluster.getNameNode(2);
+ NameNode nn0 = cluster.getNameNode(0);
+ NameNode nn1 = cluster.getNameNode(1);
+ NameNode nn2 = cluster.getNameNode(2);
- // All namenodes are in standby by default
- assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState());
- assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn1.getNameNodeState());
- assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn2.getNameNodeState());
+ // All namenodes are in standby by default
+ assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState());
+ assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn1.getNameNodeState());
+ assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn2.getNameNodeState());
- // Transition nn0 to be active
- cluster.transitionToActive(0);
- assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn0.getNameNodeState());
+ // Transition nn0 to be active
+ cluster.transitionToActive(0);
+ assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn0.getNameNodeState());
- // Transition nn1 to be active
- cluster.transitionToStandby(0);
- cluster.transitionToActive(1);
- assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState());
- assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn1.getNameNodeState());
+ // Transition nn1 to be active
+ cluster.transitionToStandby(0);
+ cluster.transitionToActive(1);
+ assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState());
+ assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn1.getNameNodeState());
- // Transition nn2 to observer
- cluster.transitionToObserver(2);
- assertEquals(HAServiceProtocol.HAServiceState.OBSERVER.ordinal(), nn2.getNameNodeState());
+ // Transition nn2 to observer
+ cluster.transitionToObserver(2);
+ assertEquals(HAServiceProtocol.HAServiceState.OBSERVER.ordinal(), nn2.getNameNodeState());
- // Shutdown nn2. Now getNameNodeState should return the INITIALIZING state.
- cluster.shutdownNameNode(2);
- assertEquals(HAServiceProtocol.HAServiceState.INITIALIZING.ordinal(), nn2.getNameNodeState());
+ // Shutdown nn2. Now getNameNodeState should return the INITIALIZING state.
+ cluster.shutdownNameNode(2);
+ assertEquals(HAServiceProtocol.HAServiceState.INITIALIZING.ordinal(),
+ nn2.getNameNodeState());
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
index b22ed863ac1272..0fb31a80fbe479 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java
@@ -423,8 +423,12 @@ public void testCheckpointCancellation() throws Exception {
* mid-checkpoint during image upload from standby to active NN.
*/
@Test
- @Timeout(value = 60)
+ @Timeout(value = 300)
public void testCheckpointCancellationDuringUpload() throws Exception {
+ // 300s, not 60s: the internal waits alone (30s TransferFsImageUpload
+ // thread-death wait, two waitForStandbyToCatchUp/waitForCheckpoint
+ // rounds, three NameNode restarts, an upload throttled to 100 B/s)
+ // can exceed 60s on a loaded CI agent. Sibling tests use 300s.
// Set dfs.namenode.checkpoint.txns differently on the first NN to avoid it
// doing checkpoint when it becomes a standby
cluster.getConfiguration(0).setInt(
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index b271069768832e..eaa1a6069b6e93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -818,6 +818,7 @@ public void testTransactionSinceLastCheckpointMetrics() throws Exception {
Random random = new Random();
int retryCount = 0;
while (retryCount < 5) {
+ MiniDFSCluster cluster2 = null;
try {
int basePort = 10060 + random.nextInt(100) * 2;
MiniDFSNNTopology topology = new MiniDFSNNTopology()
@@ -836,7 +837,7 @@ public void testTransactionSinceLastCheckpointMetrics() throws Exception {
1);
// Poll and follow ANN txns very often, for purpose of testing.
conf2.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
- MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf2)
+ cluster2 = new MiniDFSCluster.Builder(conf2)
.nnTopology(topology).numDataNodes(1).build();
cluster2.waitActive();
DistributedFileSystem fs2 = cluster2.getFileSystem(0);
@@ -877,11 +878,14 @@ public void testTransactionSinceLastCheckpointMetrics() throws Exception {
cluster2.getNameNode(1).getNamesystem()
.getTransactionsSinceLastCheckpoint(),
"SBN failed to track 2 added txns after the ckpt.");
- cluster2.shutdown();
break;
} catch (Exception e) {
LOG.warn("Unable to set up HA cluster, exception thrown: " + e);
retryCount++;
+ } finally {
+ if (cluster2 != null) {
+ cluster2.shutdown();
+ }
}
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index bf46133868279e..de0c05d397f7f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -459,67 +459,70 @@ public void testAllocShm() throws Exception {
Configuration conf = createShortCircuitConf("testAllocShm", sockDir);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
- final ShortCircuitCache cache =
- fs.getClient().getClientContext().getShortCircuitCache(0);
- cache.getDfsClientShmManager().visit(new Visitor() {
- @Override
- public void visit(HashMap info)
- throws IOException {
- // The ClientShmManager starts off empty
- assertEquals(0, info.size());
- }
- });
- DomainPeer peer = getDomainPeerToDn(conf);
- MutableBoolean usedPeer = new MutableBoolean(false);
- ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
- final DatanodeInfo datanode = new DatanodeInfoBuilder()
- .setNodeID(cluster.getDataNodes().get(0).getDatanodeId())
- .build();
- // Allocating the first shm slot requires using up a peer.
- Slot slot = cache.allocShmSlot(datanode, peer, usedPeer,
- blockId, "testAllocShm_client");
- assertNotNull(slot);
- assertTrue(usedPeer.booleanValue());
- cache.getDfsClientShmManager().visit(new Visitor() {
- @Override
- public void visit(HashMap info)
- throws IOException {
- // The ClientShmManager starts off empty
- assertEquals(1, info.size());
- PerDatanodeVisitorInfo vinfo = info.get(datanode);
- assertFalse(vinfo.disabled);
- assertEquals(0, vinfo.full.size());
- assertEquals(1, vinfo.notFull.size());
- }
- });
- cache.scheduleSlotReleaser(slot);
- // Wait for the slot to be released, and the shared memory area to be
- // closed. Since we didn't register this shared memory segment on the
- // server, it will also be a test of how well the server deals with
- // bogus client behavior.
- GenericTestUtils.waitFor(new Supplier() {
- @Override
- public Boolean get() {
- final MutableBoolean done = new MutableBoolean(false);
- try {
- cache.getDfsClientShmManager().visit(new Visitor() {
- @Override
- public void visit(HashMap info)
- throws IOException {
- done.setValue(info.get(datanode).full.isEmpty() &&
- info.get(datanode).notFull.isEmpty());
- }
- });
- } catch (IOException e) {
- LOG.error("error running visitor", e);
+ try {
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache(0);
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap info)
+ throws IOException {
+ // The ClientShmManager starts off empty
+ assertEquals(0, info.size());
}
- return done.booleanValue();
- }
- }, 10, 60000);
- cluster.shutdown();
- sockDir.close();
+ });
+ DomainPeer peer = getDomainPeerToDn(conf);
+ MutableBoolean usedPeer = new MutableBoolean(false);
+ ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
+ final DatanodeInfo datanode = new DatanodeInfoBuilder()
+ .setNodeID(cluster.getDataNodes().get(0).getDatanodeId())
+ .build();
+ // Allocating the first shm slot requires using up a peer.
+ Slot slot = cache.allocShmSlot(datanode, peer, usedPeer,
+ blockId, "testAllocShm_client");
+ assertNotNull(slot);
+ assertTrue(usedPeer.booleanValue());
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap info)
+ throws IOException {
+ // The ClientShmManager starts off empty
+ assertEquals(1, info.size());
+ PerDatanodeVisitorInfo vinfo = info.get(datanode);
+ assertFalse(vinfo.disabled);
+ assertEquals(0, vinfo.full.size());
+ assertEquals(1, vinfo.notFull.size());
+ }
+ });
+ cache.scheduleSlotReleaser(slot);
+ // Wait for the slot to be released, and the shared memory area to be
+ // closed. Since we didn't register this shared memory segment on the
+ // server, it will also be a test of how well the server deals with
+ // bogus client behavior.
+ GenericTestUtils.waitFor(new Supplier() {
+ @Override
+ public Boolean get() {
+ final MutableBoolean done = new MutableBoolean(false);
+ try {
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap info)
+ throws IOException {
+ done.setValue(info.get(datanode).full.isEmpty() &&
+ info.get(datanode).notFull.isEmpty());
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("error running visitor", e);
+ }
+ return done.booleanValue();
+ }
+ }, 10, 60000);
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
@Test
@@ -530,51 +533,54 @@ public void testShmBasedStaleness() throws Exception {
Configuration conf = createShortCircuitConf("testShmBasedStaleness", sockDir);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
- final ShortCircuitCache cache =
- fs.getClient().getClientContext().getShortCircuitCache(0);
- String TEST_FILE = "/test_file";
- final int TEST_FILE_LEN = 8193;
- final int SEED = 0xFADED;
- DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
- (short)1, SEED);
- FSDataInputStream fis = fs.open(new Path(TEST_FILE));
- int first = fis.read();
- final ExtendedBlock block =
- DFSTestUtil.getFirstBlock(fs, new Path(TEST_FILE));
- assertTrue(first != -1);
- cache.accept(new CacheVisitor() {
- @Override
- public void visit(int numOutstandingMmaps,
- Map replicas,
- Map failedLoads,
- LinkedMap evictable,
- LinkedMap evictableMmapped) {
- ShortCircuitReplica replica = replicas.get(
- ExtendedBlockId.fromExtendedBlock(block));
- assertNotNull(replica);
- assertTrue(replica.getSlot().isValid());
- }
- });
- // Stop the Namenode. This will close the socket keeping the client's
- // shared memory segment alive, and make it stale.
- cluster.getDataNodes().get(0).shutdown();
- cache.accept(new CacheVisitor() {
- @Override
- public void visit(int numOutstandingMmaps,
- Map replicas,
- Map failedLoads,
- LinkedMap evictable,
- LinkedMap evictableMmapped) {
- ShortCircuitReplica replica = replicas.get(
- ExtendedBlockId.fromExtendedBlock(block));
- assertNotNull(replica);
- assertFalse(replica.getSlot().isValid());
- }
- });
- cluster.shutdown();
- sockDir.close();
+ try {
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache(0);
+ String TEST_FILE = "/test_file";
+ final int TEST_FILE_LEN = 8193;
+ final int SEED = 0xFADED;
+ DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN,
+ (short)1, SEED);
+ FSDataInputStream fis = fs.open(new Path(TEST_FILE));
+ int first = fis.read();
+ final ExtendedBlock block =
+ DFSTestUtil.getFirstBlock(fs, new Path(TEST_FILE));
+ assertTrue(first != -1);
+ cache.accept(new CacheVisitor() {
+ @Override
+ public void visit(int numOutstandingMmaps,
+ Map replicas,
+ Map failedLoads,
+ LinkedMap evictable,
+ LinkedMap evictableMmapped) {
+ ShortCircuitReplica replica = replicas.get(
+ ExtendedBlockId.fromExtendedBlock(block));
+ assertNotNull(replica);
+ assertTrue(replica.getSlot().isValid());
+ }
+ });
+ // Stop the Namenode. This will close the socket keeping the client's
+ // shared memory segment alive, and make it stale.
+ cluster.getDataNodes().get(0).shutdown();
+ cache.accept(new CacheVisitor() {
+ @Override
+ public void visit(int numOutstandingMmaps,
+ Map replicas,
+ Map failedLoads,
+ LinkedMap evictable,
+ LinkedMap evictableMmapped) {
+ ShortCircuitReplica replica = replicas.get(
+ ExtendedBlockId.fromExtendedBlock(block));
+ assertNotNull(replica);
+ assertFalse(replica.getSlot().isValid());
+ }
+ });
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
/**
@@ -595,82 +601,85 @@ public void testUnlinkingReplicasInFileDescriptorCache() throws Exception {
1000000000L);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
- final ShortCircuitCache cache =
- fs.getClient().getClientContext().getShortCircuitCache(0);
- cache.getDfsClientShmManager().visit(new Visitor() {
- @Override
- public void visit(HashMap info)
- throws IOException {
- // The ClientShmManager starts off empty.
- assertEquals(0, info.size());
- }
- });
- final Path TEST_PATH = new Path("/test_file");
- final int TEST_FILE_LEN = 8193;
- final int SEED = 0xFADE0;
- DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN,
- (short)1, SEED);
- byte contents[] = DFSTestUtil.readFileBuffer(fs, TEST_PATH);
- byte expected[] = DFSTestUtil.
- calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
- assertTrue(Arrays.equals(contents, expected));
- // Loading this file brought the ShortCircuitReplica into our local
- // replica cache.
- final DatanodeInfo datanode = new DatanodeInfoBuilder()
- .setNodeID(cluster.getDataNodes().get(0).getDatanodeId())
- .build();
- cache.getDfsClientShmManager().visit(new Visitor() {
- @Override
- public void visit(HashMap info)
- throws IOException {
- assertTrue(info.get(datanode).full.isEmpty());
- assertFalse(info.get(datanode).disabled);
- assertEquals(1, info.get(datanode).notFull.values().size());
- DfsClientShm shm =
- info.get(datanode).notFull.values().iterator().next();
- assertFalse(shm.isDisconnected());
- }
- });
- // Remove the file whose blocks we just read.
- fs.delete(TEST_PATH, false);
+ try {
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache(0);
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap info)
+ throws IOException {
+ // The ClientShmManager starts off empty.
+ assertEquals(0, info.size());
+ }
+ });
+ final Path TEST_PATH = new Path("/test_file");
+ final int TEST_FILE_LEN = 8193;
+ final int SEED = 0xFADE0;
+ DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN,
+ (short)1, SEED);
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, TEST_PATH);
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ assertTrue(Arrays.equals(contents, expected));
+ // Loading this file brought the ShortCircuitReplica into our local
+ // replica cache.
+ final DatanodeInfo datanode = new DatanodeInfoBuilder()
+ .setNodeID(cluster.getDataNodes().get(0).getDatanodeId())
+ .build();
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap info)
+ throws IOException {
+ assertTrue(info.get(datanode).full.isEmpty());
+ assertFalse(info.get(datanode).disabled);
+ assertEquals(1, info.get(datanode).notFull.values().size());
+ DfsClientShm shm =
+ info.get(datanode).notFull.values().iterator().next();
+ assertFalse(shm.isDisconnected());
+ }
+ });
+ // Remove the file whose blocks we just read.
+ fs.delete(TEST_PATH, false);
- // Wait for the replica to be purged from the DFSClient's cache.
- GenericTestUtils.waitFor(new Supplier() {
- MutableBoolean done = new MutableBoolean(true);
- @Override
- public Boolean get() {
- try {
- done.setValue(true);
- cache.getDfsClientShmManager().visit(new Visitor() {
- @Override
- public void visit(HashMap info) throws IOException {
- assertTrue(info.get(datanode).full.isEmpty());
- assertFalse(info.get(datanode).disabled);
- assertEquals(1,
- info.get(datanode).notFull.values().size());
- DfsClientShm shm = info.get(datanode).notFull.values().
- iterator().next();
- // Check that all slots have been invalidated.
- for (Iterator iter = shm.slotIterator();
- iter.hasNext(); ) {
- Slot slot = iter.next();
- if (slot.isValid()) {
- done.setValue(false);
+ // Wait for the replica to be purged from the DFSClient's cache.
+ GenericTestUtils.waitFor(new Supplier() {
+ MutableBoolean done = new MutableBoolean(true);
+ @Override
+ public Boolean get() {
+ try {
+ done.setValue(true);
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap info) throws IOException {
+ assertTrue(info.get(datanode).full.isEmpty());
+ assertFalse(info.get(datanode).disabled);
+ assertEquals(1,
+ info.get(datanode).notFull.values().size());
+ DfsClientShm shm = info.get(datanode).notFull.values().
+ iterator().next();
+ // Check that all slots have been invalidated.
+ for (Iterator iter = shm.slotIterator();
+ iter.hasNext(); ) {
+ Slot slot = iter.next();
+ if (slot.isValid()) {
+ done.setValue(false);
+ }
}
}
- }
- });
- } catch (IOException e) {
- LOG.error("error running visitor", e);
+ });
+ } catch (IOException e) {
+ LOG.error("error running visitor", e);
+ }
+ return done.booleanValue();
}
- return done.booleanValue();
- }
- }, 10, 60000);
- cluster.shutdown();
- sockDir.close();
+ }, 10, 60000);
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
static private void checkNumberOfSegmentsAndSlots(final int expectedSegments,
@@ -713,34 +722,37 @@ public void testDataXceiverCleansUpSlotsOnFailure() throws Exception {
1000000000L);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
- final Path TEST_PATH1 = new Path("/test_file1");
- final Path TEST_PATH2 = new Path("/test_file2");
- final int TEST_FILE_LEN = 4096;
- final int SEED = 0xFADE1;
- DFSTestUtil.createFile(fs, TEST_PATH1, TEST_FILE_LEN,
- (short)1, SEED);
- DFSTestUtil.createFile(fs, TEST_PATH2, TEST_FILE_LEN,
- (short)1, SEED);
-
- // The first read should allocate one shared memory segment and slot.
- DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
-
- // The second read should fail, and we should only have 1 segment and 1 slot
- // left.
- BlockReaderFactory.setFailureInjectorForTesting(
- new TestCleanupFailureInjector());
try {
- DFSTestUtil.readFileBuffer(fs, TEST_PATH2);
- } catch (Throwable t) {
- GenericTestUtils.assertExceptionContains("TCP reads were disabled for " +
- "testing, but we failed to do a non-TCP read.", t);
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ final Path TEST_PATH1 = new Path("/test_file1");
+ final Path TEST_PATH2 = new Path("/test_file2");
+ final int TEST_FILE_LEN = 4096;
+ final int SEED = 0xFADE1;
+ DFSTestUtil.createFile(fs, TEST_PATH1, TEST_FILE_LEN,
+ (short)1, SEED);
+ DFSTestUtil.createFile(fs, TEST_PATH2, TEST_FILE_LEN,
+ (short)1, SEED);
+
+ // The first read should allocate one shared memory segment and slot.
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
+
+ // The second read should fail, and we should only have 1 segment and 1 slot
+ // left.
+ BlockReaderFactory.setFailureInjectorForTesting(
+ new TestCleanupFailureInjector());
+ try {
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH2);
+ } catch (Throwable t) {
+ GenericTestUtils.assertExceptionContains("TCP reads were disabled for " +
+ "testing, but we failed to do a non-TCP read.", t);
+ }
+ checkNumberOfSegmentsAndSlots(1, 1,
+ cluster.getDataNodes().get(0).getShortCircuitRegistry());
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
}
- checkNumberOfSegmentsAndSlots(1, 1,
- cluster.getDataNodes().get(0).getShortCircuitRegistry());
- cluster.shutdown();
- sockDir.close();
}
// Regression test for HADOOP-11802
@@ -756,51 +768,53 @@ public void testDataXceiverHandlesRequestShortCircuitShmFailure()
1000000000L);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
- final Path TEST_PATH1 = new Path("/test_file1");
- DFSTestUtil.createFile(fs, TEST_PATH1, 4096,
- (short)1, 0xFADE1);
- LOG.info("Setting failure injector and performing a read which " +
- "should fail...");
- DataNodeFaultInjector failureInjector = Mockito.mock(DataNodeFaultInjector.class);
- Mockito.doAnswer(new Answer() {
- @Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- throw new IOException("injected error into sendShmResponse");
- }
- }).when(failureInjector).sendShortCircuitShmResponse();
- DataNodeFaultInjector prevInjector = DataNodeFaultInjector.get();
- DataNodeFaultInjector.set(failureInjector);
-
try {
- // The first read will try to allocate a shared memory segment and slot.
- // The shared memory segment allocation will fail because of the failure
- // injector.
- DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
- fail("expected readFileBuffer to fail, but it succeeded.");
- } catch (Throwable t) {
- GenericTestUtils.assertExceptionContains("TCP reads were disabled for " +
- "testing, but we failed to do a non-TCP read.", t);
- }
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ final Path TEST_PATH1 = new Path("/test_file1");
+ DFSTestUtil.createFile(fs, TEST_PATH1, 4096,
+ (short)1, 0xFADE1);
+ LOG.info("Setting failure injector and performing a read which " +
+ "should fail...");
+ DataNodeFaultInjector failureInjector = Mockito.mock(DataNodeFaultInjector.class);
+ Mockito.doAnswer(new Answer() {
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ throw new IOException("injected error into sendShmResponse");
+ }
+ }).when(failureInjector).sendShortCircuitShmResponse();
+ DataNodeFaultInjector prevInjector = DataNodeFaultInjector.get();
+ DataNodeFaultInjector.set(failureInjector);
- checkNumberOfSegmentsAndSlots(0, 0,
- cluster.getDataNodes().get(0).getShortCircuitRegistry());
+ try {
+ // The first read will try to allocate a shared memory segment and slot.
+ // The shared memory segment allocation will fail because of the failure
+ // injector.
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
+ fail("expected readFileBuffer to fail, but it succeeded.");
+ } catch (Throwable t) {
+ GenericTestUtils.assertExceptionContains("TCP reads were disabled for " +
+ "testing, but we failed to do a non-TCP read.", t);
+ }
- LOG.info("Clearing failure injector and performing another read...");
- DataNodeFaultInjector.set(prevInjector);
+ checkNumberOfSegmentsAndSlots(0, 0,
+ cluster.getDataNodes().get(0).getShortCircuitRegistry());
- fs.getClient().getClientContext().getDomainSocketFactory().clearPathMap();
+ LOG.info("Clearing failure injector and performing another read...");
+ DataNodeFaultInjector.set(prevInjector);
- // The second read should succeed.
- DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
+ fs.getClient().getClientContext().getDomainSocketFactory().clearPathMap();
- // We should have added a new short-circuit shared memory segment and slot.
- checkNumberOfSegmentsAndSlots(1, 1,
- cluster.getDataNodes().get(0).getShortCircuitRegistry());
+ // The second read should succeed.
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
- cluster.shutdown();
- sockDir.close();
+ // We should have added a new short-circuit shared memory segment and slot.
+ checkNumberOfSegmentsAndSlots(1, 1,
+ cluster.getDataNodes().get(0).getShortCircuitRegistry());
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
public static class TestPreReceiptVerificationFailureInjector
@@ -824,20 +838,23 @@ public void testPreReceiptVerificationDfsClientCanDoScr() throws Exception {
1000000000L);
MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- DistributedFileSystem fs = cluster.getFileSystem();
- BlockReaderFactory.setFailureInjectorForTesting(
- new TestPreReceiptVerificationFailureInjector());
- final Path TEST_PATH1 = new Path("/test_file1");
- DFSTestUtil.createFile(fs, TEST_PATH1, 4096, (short)1, 0xFADE2);
- final Path TEST_PATH2 = new Path("/test_file2");
- DFSTestUtil.createFile(fs, TEST_PATH2, 4096, (short)1, 0xFADE2);
- DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
- DFSTestUtil.readFileBuffer(fs, TEST_PATH2);
- checkNumberOfSegmentsAndSlots(1, 2,
- cluster.getDataNodes().get(0).getShortCircuitRegistry());
- cluster.shutdown();
- sockDir.close();
+ try {
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ BlockReaderFactory.setFailureInjectorForTesting(
+ new TestPreReceiptVerificationFailureInjector());
+ final Path TEST_PATH1 = new Path("/test_file1");
+ DFSTestUtil.createFile(fs, TEST_PATH1, 4096, (short)1, 0xFADE2);
+ final Path TEST_PATH2 = new Path("/test_file2");
+ DFSTestUtil.createFile(fs, TEST_PATH2, 4096, (short)1, 0xFADE2);
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH2);
+ checkNumberOfSegmentsAndSlots(1, 2,
+ cluster.getDataNodes().get(0).getShortCircuitRegistry());
+ } finally {
+ cluster.shutdown();
+ sockDir.close();
+ }
}
@Test
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
index 50a401fe236c77..0c9695f4aec4ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
@@ -57,6 +57,7 @@
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -80,6 +81,13 @@ public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest {
}
}
+ @AfterAll
+ public static void shutdownCluster() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
@BeforeEach
public void setUp() throws Exception {
//get file system as a non-superuser