From 3e2de229624d413aa2e1572386242819b267eeea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Luis=20L=C3=B3pez?= Date: Tue, 4 Aug 2026 08:23:45 +0200 Subject: [PATCH 1/4] HDFS-17957. Shut down leaked clusters and fix the fork-pool assertion race in TestFsVolumeList. getAddReplicaForkPoolSize() now returns getParallelism() instead of the lazily grown getPoolSize(). It is @VisibleForTesting with no production callers. Contains content generated by Claude Code. Generated-by: Claude Code (Opus 5) Co-Authored-By: Claude Opus 5 --- .../fsdataset/impl/BlockPoolSlice.java | 8 +- .../fsdataset/impl/TestFsVolumeList.java | 219 +++++++++--------- 2 files changed, 118 insertions(+), 109 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java index 8c643e9e16acef..fe4bd3e121dd39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java @@ -1126,11 +1126,15 @@ protected void compute() { } /** - * Return the size of fork pool used for adding replica in map. + * Return the configured parallelism of the fork pool used for adding + * replica in map. Deliberately not {@link ForkJoinPool#getPoolSize()}: + * that reports the worker threads currently started, which the pool + * grows lazily and shrinks again when idle, so asserting on it races + * against the pool's own thread management. */ @VisibleForTesting public static int getAddReplicaForkPoolSize() { - return addReplicaThreadPool.getPoolSize(); + return addReplicaThreadPool.getParallelism(); } @VisibleForTesting diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java index 6c00e9690bb91f..53c1e15ff7ea45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java @@ -379,47 +379,51 @@ public void testAddRplicaProcessorForAddingReplicaInMap() throws Exception { cnf.setInt( DFSConfigKeys.DFS_DATANODE_VOLUMES_REPLICA_ADD_THREADPOOL_SIZE_KEY, poolSize); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(cnf).numDataNodes(1) - .storagesPerDatanode(1).build(); - DistributedFileSystem fs = cluster.getFileSystem(); - // Generate data blocks. - ExecutorService pool = Executors.newFixedThreadPool(10); - List> futureList = new ArrayList<>(); - for (int i = 0; i < 100; i++) { - Thread thread = new Thread() { - @Override - public void run() { - for (int j = 0; j < 10; j++) { - try { - DFSTestUtil.createFile(fs, new Path("File_" + getName() + j), 10, - (short) 1, 0); - } catch (IllegalArgumentException | IOException e) { - e.printStackTrace(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(cnf) + .numDataNodes(1).storagesPerDatanode(1).build()) { + DistributedFileSystem fs = cluster.getFileSystem(); + // Generate data blocks. + ExecutorService pool = Executors.newFixedThreadPool(10); + List> futureList = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + Thread thread = new Thread() { + @Override + public void run() { + for (int j = 0; j < 10; j++) { + try { + DFSTestUtil.createFile(fs, new Path("File_" + getName() + j), + 10, (short) 1, 0); + } catch (IllegalArgumentException | IOException e) { + e.printStackTrace(); + } } } - } - }; - thread.setName("FileWriter" + i); - futureList.add(pool.submit(thread)); - } - // Wait for data generation - for (Future f : futureList) { - f.get(); + }; + thread.setName("FileWriter" + i); + futureList.add(pool.submit(thread)); + } + // Wait for data generation + for (Future f : futureList) { + f.get(); + } + pool.shutdown(); + fs.close(); + FsDatasetImpl fsDataset = (FsDatasetImpl) cluster.getDataNodes().get(0) + .getFSDataset(); + ReplicaMap volumeMap = + new ReplicaMap(fsDataset.acquireDatasetLockManager()); + RamDiskReplicaTracker ramDiskReplicaMap = RamDiskReplicaTracker + .getInstance(conf, fsDataset); + FsVolumeImpl vol = (FsVolumeImpl) fsDataset.getFsVolumeReferences().get(0); + String bpid = cluster.getNamesystem().getBlockPoolId(); + // It will create BlockPoolSlice.AddReplicaProcessor task's and lunch in + // ForkJoinPool recursively + vol.getVolumeMap(bpid, volumeMap, ramDiskReplicaMap); + assertTrue(volumeMap.replicas(bpid).size() == 1000, + "Failed to add all the replica to map"); + assertEquals(poolSize, BlockPoolSlice.getAddReplicaForkPoolSize(), + "Fork pool should be initialize with configured pool size"); } - fs.close(); - FsDatasetImpl fsDataset = (FsDatasetImpl) cluster.getDataNodes().get(0) - .getFSDataset(); - ReplicaMap volumeMap = new ReplicaMap(fsDataset.acquireDatasetLockManager()); - RamDiskReplicaTracker ramDiskReplicaMap = RamDiskReplicaTracker - .getInstance(conf, fsDataset); - FsVolumeImpl vol = (FsVolumeImpl) fsDataset.getFsVolumeReferences().get(0); - String bpid = cluster.getNamesystem().getBlockPoolId(); - // It will create BlockPoolSlice.AddReplicaProcessor task's and lunch in - // ForkJoinPool recursively - vol.getVolumeMap(bpid, volumeMap, ramDiskReplicaMap); - assertTrue(volumeMap.replicas(bpid).size() == 1000, "Failed to add all the replica to map"); - assertEquals(poolSize, BlockPoolSlice.getAddReplicaForkPoolSize(), - "Fork pool should be initialize with configured pool size"); } @Test @@ -648,77 +652,78 @@ public void testExcludeSlowDiskWhenChoosingVolume() throws Exception { } } - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .hosts(hostnames) .numDataNodes(NUM_DATANODES) .storagesPerDatanode(STORAGES_PER_DATANODE) - .storageCapacities(capacities).build(); - cluster.waitActive(); - FileSystem fs = cluster.getFileSystem(); - - // Create file for each datanode. - ArrayList dataNodes = cluster.getDataNodes(); - DataNode dn0 = dataNodes.get(0); - DataNode dn1 = dataNodes.get(1); - DataNode dn2 = dataNodes.get(2); - - // Mock the first disk of each datanode is a slowest disk. - String slowDisk0OnDn0 = dn0.getFSDataset().getFsVolumeReferences().getReference(0) - .getVolume().getBaseURI().getPath(); - String slowDisk0OnDn1 = dn1.getFSDataset().getFsVolumeReferences().getReference(0) - .getVolume().getBaseURI().getPath(); - String slowDisk0OnDn2 = dn2.getFSDataset().getFsVolumeReferences().getReference(0) - .getVolume().getBaseURI().getPath(); - - String slowDisk1OnDn0 = dn0.getFSDataset().getFsVolumeReferences().getReference(1) - .getVolume().getBaseURI().getPath(); - String slowDisk1OnDn1 = dn1.getFSDataset().getFsVolumeReferences().getReference(1) - .getVolume().getBaseURI().getPath(); - String slowDisk1OnDn2 = dn2.getFSDataset().getFsVolumeReferences().getReference(1) - .getVolume().getBaseURI().getPath(); - - dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn0, ImmutableMap.of( - SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5, - SlowDiskReports.DiskOp.METADATA, 2.0)); - dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn1, ImmutableMap.of( - SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5, - SlowDiskReports.DiskOp.METADATA, 2.0)); - dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn2, ImmutableMap.of( - SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5, - SlowDiskReports.DiskOp.METADATA, 2.0)); - - dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn0, ImmutableMap.of( - SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0, - SlowDiskReports.DiskOp.METADATA, 1.0)); - dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn1, ImmutableMap.of( - SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0, - SlowDiskReports.DiskOp.METADATA, 1.0)); - dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn2, ImmutableMap.of( - SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0, - SlowDiskReports.DiskOp.METADATA, 1.0)); - - // Wait until the data on the slow disk is collected successfully. - GenericTestUtils.waitFor(new Supplier() { - @Override public Boolean get() { - return dn0.getDiskMetrics().getSlowDisksToExclude().size() == 1 && - dn1.getDiskMetrics().getSlowDisksToExclude().size() == 1 && - dn2.getDiskMetrics().getSlowDisksToExclude().size() == 1; - } - }, 1000, 5000); - - // Create a file with 3 replica. - DFSTestUtil.createFile(fs, new Path("/file0"), false, BUFFER_LENGTH, 1000, - DEFAULT_BLOCK_SIZE, (short) 3, 0, false, null); - - // Asserts that the number of blocks created on a slow disk is 0. - assertEquals(0, - dn0.getVolumeReport().stream().filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn0)) - .collect(Collectors.toList()).get(0).getNumBlocks()); - assertEquals(0, - dn1.getVolumeReport().stream().filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn1)) - .collect(Collectors.toList()).get(0).getNumBlocks()); - assertEquals(0, - dn2.getVolumeReport().stream().filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn2)) - .collect(Collectors.toList()).get(0).getNumBlocks()); + .storageCapacities(capacities).build()) { + cluster.waitActive(); + FileSystem fs = cluster.getFileSystem(); + + // Create file for each datanode. + ArrayList dataNodes = cluster.getDataNodes(); + DataNode dn0 = dataNodes.get(0); + DataNode dn1 = dataNodes.get(1); + DataNode dn2 = dataNodes.get(2); + + // Mock the first disk of each datanode is a slowest disk. + String slowDisk0OnDn0 = dn0.getFSDataset().getFsVolumeReferences() + .getReference(0).getVolume().getBaseURI().getPath(); + String slowDisk0OnDn1 = dn1.getFSDataset().getFsVolumeReferences() + .getReference(0).getVolume().getBaseURI().getPath(); + String slowDisk0OnDn2 = dn2.getFSDataset().getFsVolumeReferences() + .getReference(0).getVolume().getBaseURI().getPath(); + + String slowDisk1OnDn0 = dn0.getFSDataset().getFsVolumeReferences() + .getReference(1).getVolume().getBaseURI().getPath(); + String slowDisk1OnDn1 = dn1.getFSDataset().getFsVolumeReferences() + .getReference(1).getVolume().getBaseURI().getPath(); + String slowDisk1OnDn2 = dn2.getFSDataset().getFsVolumeReferences() + .getReference(1).getVolume().getBaseURI().getPath(); + + dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn0, ImmutableMap.of( + SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5, + SlowDiskReports.DiskOp.METADATA, 2.0)); + dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn1, ImmutableMap.of( + SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5, + SlowDiskReports.DiskOp.METADATA, 2.0)); + dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk0OnDn2, ImmutableMap.of( + SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.5, + SlowDiskReports.DiskOp.METADATA, 2.0)); + + dn0.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn0, ImmutableMap.of( + SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0, + SlowDiskReports.DiskOp.METADATA, 1.0)); + dn1.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn1, ImmutableMap.of( + SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0, + SlowDiskReports.DiskOp.METADATA, 1.0)); + dn2.getDiskMetrics().addSlowDiskForTesting(slowDisk1OnDn2, ImmutableMap.of( + SlowDiskReports.DiskOp.READ, 1.0, SlowDiskReports.DiskOp.WRITE, 1.0, + SlowDiskReports.DiskOp.METADATA, 1.0)); + + // Wait until the data on the slow disk is collected successfully. + GenericTestUtils.waitFor(new Supplier() { + @Override public Boolean get() { + return dn0.getDiskMetrics().getSlowDisksToExclude().size() == 1 && + dn1.getDiskMetrics().getSlowDisksToExclude().size() == 1 && + dn2.getDiskMetrics().getSlowDisksToExclude().size() == 1; + } + }, 1000, 5000); + + // Create a file with 3 replica. + DFSTestUtil.createFile(fs, new Path("/file0"), false, BUFFER_LENGTH, 1000, + DEFAULT_BLOCK_SIZE, (short) 3, 0, false, null); + + // Asserts that the number of blocks created on a slow disk is 0. + assertEquals(0, dn0.getVolumeReport().stream() + .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn0)) + .collect(Collectors.toList()).get(0).getNumBlocks()); + assertEquals(0, dn1.getVolumeReport().stream() + .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn1)) + .collect(Collectors.toList()).get(0).getNumBlocks()); + assertEquals(0, dn2.getVolumeReport().stream() + .filter(v -> (v.getPath() + "/").equals(slowDisk0OnDn2)) + .collect(Collectors.toList()).get(0).getNumBlocks()); + } } } From 88fe2b6c9b2598314f47413c3a44aa80065a24b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Luis=20L=C3=B3pez?= Date: Tue, 4 Aug 2026 09:01:55 +0200 Subject: [PATCH 2/4] HDFS-17957. Shut down leaked mini-cluster instances in hadoop-hdfs-project tests. 65 cases in 41 files: hadoop-hdfs (36), hadoop-hdfs-rbf (4), hadoop-hdfs-nfs (1). Test-only. Contains content generated by Claude Code. Generated-by: Claude Code (Opus 5) Co-Authored-By: Claude Opus 5 --- .../apache/hadoop/hdfs/nfs/TestMountd.java | 53 +- .../rbfbalance/TestMountTableProcedure.java | 7 +- .../federation/router/TestRouterAdmin.java | 7 +- .../router/TestRouterNamenodeHeartbeat.java | 26 +- ...estRouterAsyncRpcWhenNamenodeFailover.java | 9 + .../hadoop/fs/TestSymlinkHdfsDisable.java | 47 +- .../fs/viewfs/TestViewFsDefaultValue.java | 11 +- .../fs/viewfs/TestViewFsFileStatusHdfs.java | 13 +- .../apache/hadoop/hdfs/ListingBenchmark.java | 7 +- .../hadoop/hdfs/TestDFSAddressConfig.java | 121 ++-- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 125 +++-- .../hdfs/TestDistributedFileSystem.java | 223 ++++---- .../org/apache/hadoop/hdfs/TestGetBlocks.java | 87 +-- .../hadoop/hdfs/TestHAAuxiliaryPort.java | 85 +-- .../hadoop/hdfs/TestMiniDFSCluster.java | 4 +- .../java/org/apache/hadoop/hdfs/TestRead.java | 24 +- .../apache/hadoop/hdfs/TestReplication.java | 65 +-- .../hadoop/hdfs/TestRollingUpgrade.java | 3 +- .../client/impl/TestBlockReaderFactory.java | 511 +++++++++-------- .../impl/TestBlockReaderLocalLegacy.java | 215 +++---- .../hadoop/hdfs/qjournal/TestNNWithQJM.java | 59 +- .../TestBlockReportRateLimiting.java | 41 +- .../TestStorageBlockPoolUsageStdDev.java | 9 + .../TestBlockCountersInPendingIBR.java | 131 ++--- .../datanode/TestDataNodeInitStorage.java | 8 +- .../server/datanode/TestDataNodeMetrics.java | 70 +-- .../hdfs/server/datanode/TestHSync.java | 242 ++++---- .../datanode/TestTriggerBlockReport.java | 175 +++--- .../fsdataset/impl/TestProvidedImpl.java | 65 +-- .../fsdataset/impl/TestWriteToReplica.java | 65 +-- .../hadoop/hdfs/server/mover/TestMover.java | 73 +-- .../server/namenode/TestCacheDirectives.java | 99 ++-- .../TestCorrectnessOfQuotaAfterRenameOp.java | 9 + .../hdfs/server/namenode/TestFSImage.java | 97 ++-- .../namenode/TestNetworkTopologyServlet.java | 212 +++---- .../namenode/TestProtectedDirectories.java | 44 +- .../namenode/ha/TestBootstrapAliasmap.java | 9 + .../server/namenode/ha/TestHAMetrics.java | 50 +- .../namenode/metrics/TestNameNodeMetrics.java | 8 +- .../shortcircuit/TestShortCircuitCache.java | 527 +++++++++--------- .../web/TestWebHdfsFileSystemContract.java | 8 + 41 files changed, 1924 insertions(+), 1720 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java index 4411a4ee7ec1c5..f663565f5dc130 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestMountd.java @@ -41,34 +41,33 @@ public class TestMountd { public void testStart() throws IOException { // Start minicluster NfsConfiguration config = new NfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(config).numDataNodes(1) - .build(); - cluster.waitActive(); - - // Use emphral port in case tests are running in parallel - config.setInt("nfs3.mountd.port", 0); - config.setInt("nfs3.server.port", 0); - - int newTimeoutMillis = 1000; // 1s - // Set the new portmap rpc timeout values and check - config.setInt(NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY, - newTimeoutMillis); - assertTrue(config.getInt( - NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY, - 0) == newTimeoutMillis); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(config) + .numDataNodes(1).build()) { + cluster.waitActive(); - // Start nfs - Nfs3 nfs3 = new Nfs3(config); - nfs3.startServiceInternal(false); + // Use emphral port in case tests are running in parallel + config.setInt("nfs3.mountd.port", 0); + config.setInt("nfs3.server.port", 0); - RpcProgramMountd mountd = (RpcProgramMountd) nfs3.getMountd() - .getRpcProgram(); - mountd.nullOp(new XDR(), 1234, InetAddress.getByName("localhost")); - assertTrue(mountd.getPortmapUdpTimeoutMillis() == newTimeoutMillis); - RpcProgramNfs3 nfsd = (RpcProgramNfs3) nfs3.getRpcProgram(); - nfsd.nullProcedure(); - assertTrue(nfsd.getPortmapUdpTimeoutMillis() == newTimeoutMillis); - - cluster.shutdown(); + int newTimeoutMillis = 1000; // 1s + // Set the new portmap rpc timeout values and check + config.setInt(NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY, + newTimeoutMillis); + assertTrue(config.getInt( + NfsConfigKeys.NFS_UDP_CLIENT_PORTMAP_TIMEOUT_MILLIS_KEY, + 0) == newTimeoutMillis); + + // Start nfs + Nfs3 nfs3 = new Nfs3(config); + nfs3.startServiceInternal(false); + + RpcProgramMountd mountd = (RpcProgramMountd) nfs3.getMountd() + .getRpcProgram(); + mountd.nullOp(new XDR(), 1234, InetAddress.getByName("localhost")); + assertTrue(mountd.getPortmapUdpTimeoutMillis() == newTimeoutMillis); + RpcProgramNfs3 nfsd = (RpcProgramNfs3) nfs3.getRpcProgram(); + nfsd.nullProcedure(); + assertTrue(nfsd.getPortmapUdpTimeoutMillis() == newTimeoutMillis); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java index b305612d69e1e8..1ccdac6efcb22b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/rbfbalance/TestMountTableProcedure.java @@ -102,7 +102,12 @@ public static void globalSetUp() throws Exception { @AfterAll public static void tearDown() { - cluster.stopRouter(routerContext); + try { + cluster.stopRouter(routerContext); + } finally { + cluster.shutdown(); + cluster = null; + } } @BeforeEach diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index 8a4b06cc2f2763..0a7605a022a6dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -166,7 +166,12 @@ private static void setUpMocks() @AfterAll public static void tearDown() { - cluster.stopRouter(routerContext); + try { + cluster.stopRouter(routerContext); + } finally { + cluster.shutdown(); + cluster = null; + } } @BeforeEach diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java index 53770855236d24..51e6a6ac2b593a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java @@ -104,17 +104,21 @@ public static void tearDown() throws IOException { public void testNamenodeHeartbeatService() throws IOException { MiniRouterDFSCluster testCluster = new MiniRouterDFSCluster(true, 1); - Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration( - NAMESERVICES[0]); - NamenodeHeartbeatService server = new NamenodeHeartbeatService( - namenodeResolver, NAMESERVICES[0], NAMENODES[0]); - server.init(heartbeatConfig); - assertEquals(STATE.INITED, server.getServiceState()); - server.start(); - assertEquals(STATE.STARTED, server.getServiceState()); - server.stop(); - assertEquals(STATE.STOPPED, server.getServiceState()); - server.close(); + try { + Configuration heartbeatConfig = testCluster.generateNamenodeConfiguration( + NAMESERVICES[0]); + NamenodeHeartbeatService server = new NamenodeHeartbeatService( + namenodeResolver, NAMESERVICES[0], NAMENODES[0]); + server.init(heartbeatConfig); + assertEquals(STATE.INITED, server.getServiceState()); + server.start(); + assertEquals(STATE.STARTED, server.getServiceState()); + server.stop(); + assertEquals(STATE.STOPPED, server.getServiceState()); + server.close(); + } finally { + testCluster.shutdown(); + } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java index 64714c65d827f0..75325596f35ea8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/async/TestRouterAsyncRpcWhenNamenodeFailover.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -58,6 +59,14 @@ private void setupCluster(boolean ha) cluster.waitClusterUp(); } + @AfterEach + public void tearDown() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + @Test public void testGetFileInfoWhenNsFailover() throws Exception { setupCluster(true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java index bfdfb688aca8e7..ac34f4cbf9b211 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java @@ -40,30 +40,31 @@ public void testSymlinkHdfsDisable() throws Exception { conf.setBoolean( CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, false); // spin up minicluster, get dfs and filecontext - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - DistributedFileSystem dfs = cluster.getFileSystem(); - FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf); - // Create test files/links - FileContextTestHelper helper = new FileContextTestHelper( - "/tmp/TestSymlinkHdfsDisable"); - Path root = helper.getTestRootPath(fc); - Path target = new Path(root, "target"); - Path link = new Path(root, "link"); - DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD); - fc.createSymlink(target, link, false); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) { + DistributedFileSystem dfs = cluster.getFileSystem(); + FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf); + // Create test files/links + FileContextTestHelper helper = new FileContextTestHelper( + "/tmp/TestSymlinkHdfsDisable"); + Path root = helper.getTestRootPath(fc); + Path target = new Path(root, "target"); + Path link = new Path(root, "link"); + DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD); + fc.createSymlink(target, link, false); - // Try to resolve links with FileSystem and FileContext - try { - fc.open(link); - fail("Expected error when attempting to resolve link"); - } catch (IOException e) { - GenericTestUtils.assertExceptionContains("resolution is disabled", e); - } - try { - dfs.open(link); - fail("Expected error when attempting to resolve link"); - } catch (IOException e) { - GenericTestUtils.assertExceptionContains("resolution is disabled", e); + // Try to resolve links with FileSystem and FileContext + try { + fc.open(link); + fail("Expected error when attempting to resolve link"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("resolution is disabled", e); + } + try { + dfs.open(link); + fail("Expected error when attempting to resolve link"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("resolution is disabled", e); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java index e1881eea7e0f62..4dc7cc3edb7cc0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsDefaultValue.java @@ -220,8 +220,15 @@ public void testGetQuotaUsageWithQuotaDefined() throws IOException { @AfterAll public static void cleanup() throws IOException { - fHdfs.delete(new Path(testFileName), true); - fHdfs.delete(notInMountpointPath, true); + try { + fHdfs.delete(new Path(testFileName), true); + fHdfs.delete(notInMountpointPath, true); + } finally { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java index 3bdeb0c47b1ef4..8bd759e977bf33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsFileStatusHdfs.java @@ -118,9 +118,16 @@ public void testGetFileChecksum() throws IOException, URISyntaxException { @AfterAll public static void cleanup() throws IOException { - fHdfs.delete(new Path(testfilename), true); - fHdfs.delete(new Path(someFile), true); - fHdfs.delete(new Path(someFile + "other"), true); + try { + fHdfs.delete(new Path(testfilename), true); + fHdfs.delete(new Path(someFile), true); + fHdfs.delete(new Path(someFile + "other"), true); + } finally { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java index 67deaf3c3ed423..16711423313e62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ListingBenchmark.java @@ -26,10 +26,11 @@ public class ListingBenchmark { public static void main(String[] args) throws IOException { HdfsConfiguration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(0) .format(true) - .build(); - NameNode nn = cluster.getNameNode(); + .build()) { + NameNode nn = cluster.getNameNode(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java index cfd3c53c075756..612fe09392251a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java @@ -49,67 +49,66 @@ public void testDFSAddressConfig() throws IOException { /*------------------------------------------------------------------------- * By default, the DataNode socket address should be localhost (127.0.0.1). *------------------------------------------------------------------------*/ - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.waitActive(); - - ArrayList dns = cluster.getDataNodes(); - DataNode dn = dns.get(0); - - String selfSocketAddr = dn.getXferAddress().toString(); - System.out.println("DN Self Socket Addr == " + selfSocketAddr); - assertTrue(selfSocketAddr.contains("/127.0.0.1:")); - - /*------------------------------------------------------------------------- - * Shut down the datanodes, reconfigure, and bring them back up. - * Even if told to use the configuration properties for dfs.datanode, - * MiniDFSCluster.startDataNodes() should use localhost as the default if - * the dfs.datanode properties are not set. - *------------------------------------------------------------------------*/ - for (int i = 0; i < dns.size(); i++) { - DataNodeProperties dnp = cluster.stopDataNode(i); - assertNotNull(dnp, "Should have been able to stop simulated datanode"); - } - - conf.unset(DFS_DATANODE_ADDRESS_KEY); - conf.unset(DFS_DATANODE_HTTP_ADDRESS_KEY); - conf.unset(DFS_DATANODE_IPC_ADDRESS_KEY); - - cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR, - null, null, null, false, true); - - dns = cluster.getDataNodes(); - dn = dns.get(0); - - selfSocketAddr = dn.getXferAddress().toString(); - System.out.println("DN Self Socket Addr == " + selfSocketAddr); - // assert that default self socket address is 127.0.0.1 - assertTrue(selfSocketAddr.contains("/127.0.0.1:")); - - /*------------------------------------------------------------------------- - * Shut down the datanodes, reconfigure, and bring them back up. - * This time, modify the dfs.datanode properties and make sure that they - * are used to configure sockets by MiniDFSCluster.startDataNodes(). - *------------------------------------------------------------------------*/ - for (int i = 0; i < dns.size(); i++) { - DataNodeProperties dnp = cluster.stopDataNode(i); - assertNotNull(dnp, "Should have been able to stop simulated datanode"); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) { + cluster.waitActive(); + + ArrayList dns = cluster.getDataNodes(); + DataNode dn = dns.get(0); + + String selfSocketAddr = dn.getXferAddress().toString(); + System.out.println("DN Self Socket Addr == " + selfSocketAddr); + assertTrue(selfSocketAddr.contains("/127.0.0.1:")); + + /*----------------------------------------------------------------------- + * Shut down the datanodes, reconfigure, and bring them back up. + * Even if told to use the configuration properties for dfs.datanode, + * MiniDFSCluster.startDataNodes() should use localhost as the default if + * the dfs.datanode properties are not set. + *----------------------------------------------------------------------*/ + for (int i = 0; i < dns.size(); i++) { + DataNodeProperties dnp = cluster.stopDataNode(i); + assertNotNull(dnp, "Should have been able to stop simulated datanode"); + } + + conf.unset(DFS_DATANODE_ADDRESS_KEY); + conf.unset(DFS_DATANODE_HTTP_ADDRESS_KEY); + conf.unset(DFS_DATANODE_IPC_ADDRESS_KEY); + + cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR, + null, null, null, false, true); + + dns = cluster.getDataNodes(); + dn = dns.get(0); + + selfSocketAddr = dn.getXferAddress().toString(); + System.out.println("DN Self Socket Addr == " + selfSocketAddr); + // assert that default self socket address is 127.0.0.1 + assertTrue(selfSocketAddr.contains("/127.0.0.1:")); + + /*----------------------------------------------------------------------- + * Shut down the datanodes, reconfigure, and bring them back up. + * This time, modify the dfs.datanode properties and make sure that they + * are used to configure sockets by MiniDFSCluster.startDataNodes(). + *----------------------------------------------------------------------*/ + for (int i = 0; i < dns.size(); i++) { + DataNodeProperties dnp = cluster.stopDataNode(i); + assertNotNull(dnp, "Should have been able to stop simulated datanode"); + } + + conf.set(DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0"); + conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); + conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0"); + + cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR, + null, null, null, false, true); + + dns = cluster.getDataNodes(); + dn = dns.get(0); + + selfSocketAddr = dn.getXferAddress().toString(); + System.out.println("DN Self Socket Addr == " + selfSocketAddr); + // assert that default self socket address is 0.0.0.0 + assertTrue(selfSocketAddr.contains("/0.0.0.0:")); } - - conf.set(DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0"); - conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); - conf.set(DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0"); - - cluster.startDataNodes(conf, 1, true, StartupOption.REGULAR, - null, null, null, false, true); - - dns = cluster.getDataNodes(); - dn = dns.get(0); - - selfSocketAddr = dn.getXferAddress().toString(); - System.out.println("DN Self Socket Addr == " + selfSocketAddr); - // assert that default self socket address is 0.0.0.0 - assertTrue(selfSocketAddr.contains("/0.0.0.0:")); - - cluster.shutdown(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java index 5453a7465e86d3..51683607a25946 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java @@ -656,86 +656,89 @@ public void testPreserveEditLogs() throws Exception { */ Configuration conf = new HdfsConfiguration(); conf = UpgradeUtilities.initializeStorageStateConf(1, conf); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) .format(false) .manageDataDfsDirs(false) .manageNameDfsDirs(false) .startupOption(StartupOption.UPGRADE) - .build(); - DFSInotifyEventInputStream ieis = - cluster.getFileSystem().getInotifyEventStream(0); - - EventBatch batch; - Event.CreateEvent ce; - Event.RenameEvent re; + .build()) { + DFSInotifyEventInputStream ieis = + cluster.getFileSystem().getInotifyEventStream(0); - // mkdir /input - batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); - assertEquals(1, batch.getEvents().length); - assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); - ce = (Event.CreateEvent) batch.getEvents()[0]; - assertEquals(ce.getPath(), "/input"); + EventBatch batch; + Event.CreateEvent ce; + Event.RenameEvent re; - // mkdir /input/dir1~5 - for (int i = 1; i <= 5; i++) { + // mkdir /input batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); assertEquals(1, batch.getEvents().length); assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); ce = (Event.CreateEvent) batch.getEvents()[0]; - assertEquals(ce.getPath(), "/input/dir" + i); - } - // copyFromLocal randome_file_1~2 /input/dir1~2 - for (int i = 1; i <= 2; i++) { - batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); - assertEquals(1, batch.getEvents().length); - if (batch.getEvents()[0].getEventType() != Event.EventType.CREATE) { - FSImage.LOG.debug(""); + assertEquals(ce.getPath(), "/input"); + + // mkdir /input/dir1~5 + for (int i = 1; i <= 5; i++) { + batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); + assertEquals(1, batch.getEvents().length); + assertTrue( + batch.getEvents()[0].getEventType() == Event.EventType.CREATE); + ce = (Event.CreateEvent) batch.getEvents()[0]; + assertEquals(ce.getPath(), "/input/dir" + i); } - assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE); + // copyFromLocal randome_file_1~2 /input/dir1~2 + for (int i = 1; i <= 2; i++) { + batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); + assertEquals(1, batch.getEvents().length); + if (batch.getEvents()[0].getEventType() != Event.EventType.CREATE) { + FSImage.LOG.debug(""); + } + assertTrue( + batch.getEvents()[0].getEventType() == Event.EventType.CREATE); - // copyFromLocal randome_file_1 /input/dir1, CLOSE - batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); - assertEquals(1, batch.getEvents().length); - assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CLOSE); + // copyFromLocal randome_file_1 /input/dir1, CLOSE + batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); + assertEquals(1, batch.getEvents().length); + assertTrue( + batch.getEvents()[0].getEventType() == Event.EventType.CLOSE); - // copyFromLocal randome_file_1 /input/dir1, CLOSE + // copyFromLocal randome_file_1 /input/dir1, CLOSE + batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); + assertEquals(1, batch.getEvents().length); + assertTrue(batch.getEvents()[0].getEventType() == + Event.EventType.RENAME); + re = (Event.RenameEvent) batch.getEvents()[0]; + assertEquals(re.getDstPath(), "/input/dir" + i + "/randome_file_" + i); + } + + // mv /input/dir1/randome_file_1 /input/dir3/randome_file_3 + long txIDBeforeRename = batch.getTxid(); batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); assertEquals(1, batch.getEvents().length); - assertTrue(batch.getEvents()[0].getEventType() == - Event.EventType.RENAME); + assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME); re = (Event.RenameEvent) batch.getEvents()[0]; - assertEquals(re.getDstPath(), "/input/dir" + i + "/randome_file_" + i); - } - - // mv /input/dir1/randome_file_1 /input/dir3/randome_file_3 - long txIDBeforeRename = batch.getTxid(); - batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); - assertEquals(1, batch.getEvents().length); - assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME); - re = (Event.RenameEvent) batch.getEvents()[0]; - assertEquals(re.getDstPath(), "/input/dir3/randome_file_3"); + assertEquals(re.getDstPath(), "/input/dir3/randome_file_3"); - // rmdir /input/dir1 - batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); - assertEquals(1, batch.getEvents().length); - assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.UNLINK); - assertEquals(((Event.UnlinkEvent) batch.getEvents()[0]).getPath(), - "/input/dir1"); - long lastTxID = batch.getTxid(); - - // Start inotify from the tx before rename /input/dir1/randome_file_1 - ieis = cluster.getFileSystem().getInotifyEventStream(txIDBeforeRename); - batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); - assertEquals(1, batch.getEvents().length); - assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME); - re = (Event.RenameEvent) batch.getEvents()[0]; - assertEquals(re.getDstPath(), "/input/dir3/randome_file_3"); + // rmdir /input/dir1 + batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); + assertEquals(1, batch.getEvents().length); + assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.UNLINK); + assertEquals(((Event.UnlinkEvent) batch.getEvents()[0]).getPath(), + "/input/dir1"); + long lastTxID = batch.getTxid(); - // Try to read beyond available edits - ieis = cluster.getFileSystem().getInotifyEventStream(lastTxID + 1); - assertNull(ieis.poll()); + // Start inotify from the tx before rename /input/dir1/randome_file_1 + ieis = cluster.getFileSystem().getInotifyEventStream(txIDBeforeRename); + batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis); + assertEquals(1, batch.getEvents().length); + assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME); + re = (Event.RenameEvent) batch.getEvents()[0]; + assertEquals(re.getDstPath(), "/input/dir3/randome_file_3"); - cluster.shutdown(); + // Try to read beyond available edits + ieis = cluster.getFileSystem().getInotifyEventStream(lastTxID + 1); + assertNull(ieis.poll()); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 91bc36042e22ef..9d605eeef4f37e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -1260,133 +1260,136 @@ public void testFileChecksum() throws Exception { final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(2).build(); - final FileSystem hdfs = cluster.getFileSystem(); - - final String nnAddr = conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); - final UserGroupInformation current = UserGroupInformation.getCurrentUser(); - final UserGroupInformation ugi = UserGroupInformation.createUserForTesting( - current.getShortUserName() + "x", new String[]{"user"}); - try { - hdfs.getFileChecksum(new Path( - "/test/TestNonExistingFile")); - fail("Expecting FileNotFoundException"); - } catch (FileNotFoundException e) { - assertTrue(e.getMessage().contains("File does not exist: /test/TestNonExistingFile"), - "Not throwing the intended exception message"); - } + final FileSystem hdfs = cluster.getFileSystem(); - try { - Path path = new Path("/test/TestExistingDir/"); - hdfs.mkdirs(path); - hdfs.getFileChecksum(path); - fail("Expecting FileNotFoundException"); - } catch (FileNotFoundException e) { - assertTrue(e.getMessage().contains("Path is not a file: /test/TestExistingDir"), - "Not throwing the intended exception message"); - } - - //webhdfs - final String webhdfsuri = WebHdfsConstants.WEBHDFS_SCHEME + "://" + nnAddr; - System.out.println("webhdfsuri=" + webhdfsuri); - final FileSystem webhdfs = ugi.doAs( - new PrivilegedExceptionAction() { - @Override - public FileSystem run() throws Exception { - return new Path(webhdfsuri).getFileSystem(conf); + final String nnAddr = conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); + final UserGroupInformation current = UserGroupInformation.getCurrentUser(); + final UserGroupInformation ugi = UserGroupInformation.createUserForTesting( + current.getShortUserName() + "x", new String[]{"user"}); + + try { + hdfs.getFileChecksum(new Path( + "/test/TestNonExistingFile")); + fail("Expecting FileNotFoundException"); + } catch (FileNotFoundException e) { + assertTrue(e.getMessage().contains("File does not exist: /test/TestNonExistingFile"), + "Not throwing the intended exception message"); } - }); - - final Path dir = new Path("/filechecksum"); - final int block_size = 1024; - final int buffer_size = conf.getInt( - CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096); - conf.setInt(HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 512); - - //try different number of blocks - for(int n = 0; n < 5; n++) { - //generate random data - final byte[] data = new byte[RAN.nextInt(block_size/2-1)+n*block_size+1]; - RAN.nextBytes(data); - System.out.println("data.length=" + data.length); - - //write data to a file - final Path foo = new Path(dir, "foo" + n); - { - final FSDataOutputStream out = hdfs.create(foo, false, buffer_size, - (short)2, block_size); - out.write(data); - out.close(); + + try { + Path path = new Path("/test/TestExistingDir/"); + hdfs.mkdirs(path); + hdfs.getFileChecksum(path); + fail("Expecting FileNotFoundException"); + } catch (FileNotFoundException e) { + assertTrue(e.getMessage().contains("Path is not a file: /test/TestExistingDir"), + "Not throwing the intended exception message"); } - - //compute checksum - final FileChecksum hdfsfoocs = hdfs.getFileChecksum(foo); - System.out.println("hdfsfoocs=" + hdfsfoocs); //webhdfs - final FileChecksum webhdfsfoocs = webhdfs.getFileChecksum(foo); - System.out.println("webhdfsfoocs=" + webhdfsfoocs); + final String webhdfsuri = WebHdfsConstants.WEBHDFS_SCHEME + "://" + nnAddr; + System.out.println("webhdfsuri=" + webhdfsuri); + final FileSystem webhdfs = ugi.doAs( + new PrivilegedExceptionAction() { + @Override + public FileSystem run() throws Exception { + return new Path(webhdfsuri).getFileSystem(conf); + } + }); - final Path webhdfsqualified = new Path(webhdfsuri + dir, "foo" + n); - final FileChecksum webhdfs_qfoocs = - webhdfs.getFileChecksum(webhdfsqualified); - System.out.println("webhdfs_qfoocs=" + webhdfs_qfoocs); + final Path dir = new Path("/filechecksum"); + final int block_size = 1024; + final int buffer_size = conf.getInt( + CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096); + conf.setInt(HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 512); + + //try different number of blocks + for(int n = 0; n < 5; n++) { + //generate random data + final byte[] data = new byte[RAN.nextInt(block_size/2-1)+n*block_size+1]; + RAN.nextBytes(data); + System.out.println("data.length=" + data.length); + + //write data to a file + final Path foo = new Path(dir, "foo" + n); + { + final FSDataOutputStream out = hdfs.create(foo, false, buffer_size, + (short)2, block_size); + out.write(data); + out.close(); + } - //create a zero byte file - final Path zeroByteFile = new Path(dir, "zeroByteFile" + n); - { - final FSDataOutputStream out = hdfs.create(zeroByteFile, false, - buffer_size, (short)2, block_size); - out.close(); - } + //compute checksum + final FileChecksum hdfsfoocs = hdfs.getFileChecksum(foo); + System.out.println("hdfsfoocs=" + hdfsfoocs); - //write another file - final Path bar = new Path(dir, "bar" + n); - { - final FSDataOutputStream out = hdfs.create(bar, false, buffer_size, - (short)2, block_size); - out.write(data); - out.close(); - } + //webhdfs + final FileChecksum webhdfsfoocs = webhdfs.getFileChecksum(foo); + System.out.println("webhdfsfoocs=" + webhdfsfoocs); - { - final FileChecksum zeroChecksum = hdfs.getFileChecksum(zeroByteFile); - final String magicValue = - "MD5-of-0MD5-of-0CRC32:70bc8f4b72a86921468bf8e8441dce51"; - // verify the magic val for zero byte files - assertEquals(magicValue, zeroChecksum.toString()); - - //verify checksums for empty file and 0 request length - final FileChecksum checksumWith0 = hdfs.getFileChecksum(bar, 0); - assertEquals(zeroChecksum, checksumWith0); - - //verify checksum - final FileChecksum barcs = hdfs.getFileChecksum(bar); - final int barhashcode = barcs.hashCode(); - assertEquals(hdfsfoocs.hashCode(), barhashcode); - assertEquals(hdfsfoocs, barcs); + final Path webhdfsqualified = new Path(webhdfsuri + dir, "foo" + n); + final FileChecksum webhdfs_qfoocs = + webhdfs.getFileChecksum(webhdfsqualified); + System.out.println("webhdfs_qfoocs=" + webhdfs_qfoocs); - //webhdfs - assertEquals(webhdfsfoocs.hashCode(), barhashcode); - assertEquals(webhdfsfoocs, barcs); + //create a zero byte file + final Path zeroByteFile = new Path(dir, "zeroByteFile" + n); + { + final FSDataOutputStream out = hdfs.create(zeroByteFile, false, + buffer_size, (short)2, block_size); + out.close(); + } - assertEquals(webhdfs_qfoocs.hashCode(), barhashcode); - assertEquals(webhdfs_qfoocs, barcs); - } + //write another file + final Path bar = new Path(dir, "bar" + n); + { + final FSDataOutputStream out = hdfs.create(bar, false, buffer_size, + (short)2, block_size); + out.write(data); + out.close(); + } + + { + final FileChecksum zeroChecksum = hdfs.getFileChecksum(zeroByteFile); + final String magicValue = + "MD5-of-0MD5-of-0CRC32:70bc8f4b72a86921468bf8e8441dce51"; + // verify the magic val for zero byte files + assertEquals(magicValue, zeroChecksum.toString()); + + //verify checksums for empty file and 0 request length + final FileChecksum checksumWith0 = hdfs.getFileChecksum(bar, 0); + assertEquals(zeroChecksum, checksumWith0); + + //verify checksum + final FileChecksum barcs = hdfs.getFileChecksum(bar); + final int barhashcode = barcs.hashCode(); + assertEquals(hdfsfoocs.hashCode(), barhashcode); + assertEquals(hdfsfoocs, barcs); + + //webhdfs + assertEquals(webhdfsfoocs.hashCode(), barhashcode); + assertEquals(webhdfsfoocs, barcs); + + assertEquals(webhdfs_qfoocs.hashCode(), barhashcode); + assertEquals(webhdfs_qfoocs, barcs); + } - hdfs.setPermission(dir, new FsPermission((short)0)); + hdfs.setPermission(dir, new FsPermission((short)0)); - { //test permission error on webhdfs - try { - webhdfs.getFileChecksum(webhdfsqualified); - fail(); - } catch(IOException ioe) { - FileSystem.LOG.info("GOOD: getting an exception", ioe); + { //test permission error on webhdfs + try { + webhdfs.getFileChecksum(webhdfsqualified); + fail(); + } catch(IOException ioe) { + FileSystem.LOG.info("GOOD: getting an exception", ioe); + } } + hdfs.setPermission(dir, new FsPermission((short)0777)); } - hdfs.setPermission(dir, new FsPermission((short)0777)); + } finally { + cluster.shutdown(); } - cluster.shutdown(); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java index 3d2f286f411c4d..f6da6597598259 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java @@ -489,58 +489,59 @@ public void testReadSkipStaleStorage() throws Exception { final Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(1) .storagesPerDatanode(storageNum) - .build(); - cluster.waitActive(); + .build()) { + cluster.waitActive(); - FileSystem fs = cluster.getFileSystem(); - DFSTestUtil.createFile(fs, path, false, 1024, fileLen, - BLOCK_SIZE, repFactor, 0, true); + FileSystem fs = cluster.getFileSystem(); + DFSTestUtil.createFile(fs, path, false, 1024, fileLen, + BLOCK_SIZE, repFactor, 0, true); - // get datanode info - ClientProtocol client = NameNodeProxies.createProxy(conf, - cluster.getFileSystem(0).getUri(), - ClientProtocol.class).getProxy(); - DatanodeInfo[] dataNodes = client.getDatanodeReport(DatanodeReportType.ALL); + // get datanode info + ClientProtocol client = NameNodeProxies.createProxy(conf, + cluster.getFileSystem(0).getUri(), + ClientProtocol.class).getProxy(); + DatanodeInfo[] dataNodes = client.getDatanodeReport(DatanodeReportType.ALL); + + // get storage info + BlockManager bm0 = cluster.getNamesystem(0).getBlockManager(); + DatanodeStorageInfo[] storageInfos = bm0.getDatanodeManager() + .getDatanode(dataNodes[0].getDatanodeUuid()).getStorageInfos(); - // get storage info - BlockManager bm0 = cluster.getNamesystem(0).getBlockManager(); - DatanodeStorageInfo[] storageInfos = bm0.getDatanodeManager() - .getDatanode(dataNodes[0].getDatanodeUuid()).getStorageInfos(); + InetSocketAddress addr = new InetSocketAddress("localhost", + cluster.getNameNodePort()); + NamenodeProtocol namenode = NameNodeProxies.createProxy(conf, + DFSUtilClient.getNNUri(addr), NamenodeProtocol.class).getProxy(); - InetSocketAddress addr = new InetSocketAddress("localhost", - cluster.getNameNodePort()); - NamenodeProtocol namenode = NameNodeProxies.createProxy(conf, - DFSUtilClient.getNNUri(addr), NamenodeProtocol.class).getProxy(); - - // check blocks count equals to blockNum - BlockWithLocations[] blocks = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); - assertEquals(blockNum, blocks.length); - - // calculate the block count on storage[0] - int count = 0; - for (BlockWithLocations b : blocks) { - for (String s : b.getStorageIDs()) { - if (s.equals(storageInfos[0].getStorageID())) { - count++; + // check blocks count equals to blockNum + BlockWithLocations[] blocks = namenode.getBlocks( + dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); + assertEquals(blockNum, blocks.length); + + // calculate the block count on storage[0] + int count = 0; + for (BlockWithLocations b : blocks) { + for (String s : b.getStorageIDs()) { + if (s.equals(storageInfos[0].getStorageID())) { + count++; + } } } - } - // set storage[0] stale - storageInfos[0].setBlockContentsStale(true); - blocks = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); - assertEquals(blockNum - count, blocks.length); - - // set all storage stale - bm0.getDatanodeManager().markAllDatanodesStaleAndSetKeyUpdateIfNeed(); - blocks = namenode.getBlocks( - dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); - assertEquals(0, blocks.length); + // set storage[0] stale + storageInfos[0].setBlockContentsStale(true); + blocks = namenode.getBlocks( + dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); + assertEquals(blockNum - count, blocks.length); + + // set all storage stale + bm0.getDatanodeManager().markAllDatanodesStaleAndSetKeyUpdateIfNeed(); + blocks = namenode.getBlocks( + dataNodes[0], fileLen*2, 0, 0, null).getBlocks(); + assertEquals(0, blocks.length); + } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java index 0992f54b891901..4a0b5eda9f9919 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHAAuxiliaryPort.java @@ -52,58 +52,59 @@ public void testHAAuxiliaryPort() throws Exception { .addNN(new MiniDFSNNTopology.NNConf("nn1")) .addNN(new MiniDFSNNTopology.NNConf("nn2"))); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) .numDataNodes(0) - .build(); - cluster.transitionToActive(0); - cluster.waitActive(); + .build()) { + cluster.transitionToActive(0); + cluster.waitActive(); - NameNode nn0 = cluster.getNameNode(0); - NameNode nn1 = cluster.getNameNode(1); + NameNode nn0 = cluster.getNameNode(0); + NameNode nn1 = cluster.getNameNode(1); - // all the addresses below are valid nn0 addresses - NameNodeRpcServer rpcServer0 = (NameNodeRpcServer)nn0.getRpcServer(); - InetSocketAddress server0RpcAddress = rpcServer0.getRpcAddress(); - Set auxAddrServer0 = - rpcServer0.getAuxiliaryRpcAddresses(); - assertEquals(2, auxAddrServer0.size()); + // all the addresses below are valid nn0 addresses + NameNodeRpcServer rpcServer0 = (NameNodeRpcServer)nn0.getRpcServer(); + InetSocketAddress server0RpcAddress = rpcServer0.getRpcAddress(); + Set auxAddrServer0 = + rpcServer0.getAuxiliaryRpcAddresses(); + assertEquals(2, auxAddrServer0.size()); - // all the addresses below are valid nn1 addresses - NameNodeRpcServer rpcServer1 = (NameNodeRpcServer)nn1.getRpcServer(); - InetSocketAddress server1RpcAddress = rpcServer1.getRpcAddress(); - Set auxAddrServer1 = - rpcServer1.getAuxiliaryRpcAddresses(); - assertEquals(2, auxAddrServer1.size()); + // all the addresses below are valid nn1 addresses + NameNodeRpcServer rpcServer1 = (NameNodeRpcServer)nn1.getRpcServer(); + InetSocketAddress server1RpcAddress = rpcServer1.getRpcAddress(); + Set auxAddrServer1 = + rpcServer1.getAuxiliaryRpcAddresses(); + assertEquals(2, auxAddrServer1.size()); - // mkdir on nn0 uri 0 - URI nn0URI = new URI("hdfs://localhost:" + - server0RpcAddress.getPort()); - try (DFSClient client0 = new DFSClient(nn0URI, conf)){ - client0.mkdirs("/test", null, true); - // should be available on other ports also - for (InetSocketAddress auxAddr : auxAddrServer0) { - nn0URI = new URI("hdfs://localhost:" + auxAddr.getPort()); - try (DFSClient clientTmp = new DFSClient(nn0URI, conf)) { - assertTrue(clientTmp.exists("/test")); + // mkdir on nn0 uri 0 + URI nn0URI = new URI("hdfs://localhost:" + + server0RpcAddress.getPort()); + try (DFSClient client0 = new DFSClient(nn0URI, conf)){ + client0.mkdirs("/test", null, true); + // should be available on other ports also + for (InetSocketAddress auxAddr : auxAddrServer0) { + nn0URI = new URI("hdfs://localhost:" + auxAddr.getPort()); + try (DFSClient clientTmp = new DFSClient(nn0URI, conf)) { + assertTrue(clientTmp.exists("/test")); + } } } - } - // now perform a failover - cluster.shutdownNameNode(0); - cluster.transitionToActive(1); + // now perform a failover + cluster.shutdownNameNode(0); + cluster.transitionToActive(1); - // then try to read the file from the nn1 - URI nn1URI = new URI("hdfs://localhost:" + - server1RpcAddress.getPort()); - try (DFSClient client1 = new DFSClient(nn1URI, conf)) { - assertTrue(client1.exists("/test")); - // should be available on other ports also - for (InetSocketAddress auxAddr : auxAddrServer1) { - nn1URI = new URI("hdfs://localhost:" + auxAddr.getPort()); - try (DFSClient clientTmp = new DFSClient(nn1URI, conf)) { - assertTrue(client1.exists("/test")); + // then try to read the file from the nn1 + URI nn1URI = new URI("hdfs://localhost:" + + server1RpcAddress.getPort()); + try (DFSClient client1 = new DFSClient(nn1URI, conf)) { + assertTrue(client1.exists("/test")); + // should be available on other ports also + for (InetSocketAddress auxAddr : auxAddrServer1) { + nn1URI = new URI("hdfs://localhost:" + auxAddr.getPort()); + try (DFSClient clientTmp = new DFSClient(nn1URI, conf)) { + assertTrue(client1.exists("/test")); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java index be932c2267ac33..e04ee2855b3d8e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java @@ -225,11 +225,11 @@ public void testIsClusterUpAfterShutdown() throws Throwable { try { DistributedFileSystem dfs = cluster4.getFileSystem(); dfs.setSafeMode(SafeModeAction.ENTER); - cluster4.shutdown(); } finally { + cluster4.shutdown(); while(cluster4.isClusterUp()){ Thread.sleep(1000); - } + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java index 8f5ba9018dfa7b..70145552517a32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java @@ -96,12 +96,12 @@ public void testEOFWithBlockReaderLocal() throws Exception { try { final Configuration conf = testContext.newConfiguration(); conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_CACHE_READAHEAD, BLOCK_SIZE); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); - testEOF(cluster, 1); - testEOF(cluster, 14); - testEOF(cluster, 10000); - cluster.shutdown(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1).format(true).build()) { + testEOF(cluster, 1); + testEOF(cluster, 14); + testEOF(cluster, 10000); + } } finally { testContext.close(); } @@ -112,12 +112,12 @@ public void testEOFWithBlockReaderLocal() throws Exception { public void testEOFWithRemoteBlockReader() throws Exception { final Configuration conf = new Configuration(); conf.setLong(HdfsClientConfigKeys.DFS_CLIENT_CACHE_READAHEAD, BLOCK_SIZE); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) - .format(true).build(); - testEOF(cluster, 1); - testEOF(cluster, 14); - testEOF(cluster, 10000); - cluster.shutdown(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1).format(true).build()) { + testEOF(cluster, 1); + testEOF(cluster, 14); + testEOF(cluster, 10000); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java index 4ff9f07419fd7f..dcec782f9e6031 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java @@ -228,46 +228,49 @@ private void testBadBlockReportOnTransfer( int replicaCount = 0; short replFactor = 1; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); - fs = cluster.getFileSystem(); - dfsClient = new DFSClient(new InetSocketAddress("localhost", - cluster.getNameNodePort()), conf); + try { + cluster.waitActive(); + fs = cluster.getFileSystem(); + dfsClient = new DFSClient(new InetSocketAddress("localhost", + cluster.getNameNodePort()), conf); - // Create file with replication factor of 1 - Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1"); - DFSTestUtil.createFile(fs, file1, 1024, replFactor, 0); - DFSTestUtil.waitReplication(fs, file1, replFactor); + // Create file with replication factor of 1 + Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1"); + DFSTestUtil.createFile(fs, file1, 1024, replFactor, 0); + DFSTestUtil.waitReplication(fs, file1, replFactor); - // Corrupt the block belonging to the created file - ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1); + // Corrupt the block belonging to the created file + ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1); - int blockFilesCorrupted = - corruptBlockByDeletingBlockFile? - cluster.corruptBlockOnDataNodesByDeletingBlockFile(block) : - cluster.corruptBlockOnDataNodes(block); + int blockFilesCorrupted = + corruptBlockByDeletingBlockFile? + cluster.corruptBlockOnDataNodesByDeletingBlockFile(block) : + cluster.corruptBlockOnDataNodes(block); - assertEquals(replFactor, blockFilesCorrupted, "Corrupted too few blocks"); + assertEquals(replFactor, blockFilesCorrupted, "Corrupted too few blocks"); - // Increase replication factor, this should invoke transfer request - // Receiving datanode fails on checksum and reports it to namenode - replFactor = 2; - fs.setReplication(file1, replFactor); + // Increase replication factor, this should invoke transfer request + // Receiving datanode fails on checksum and reports it to namenode + replFactor = 2; + fs.setReplication(file1, replFactor); - // Now get block details and check if the block is corrupt - blocks = dfsClient.getNamenode(). - getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); - while (blocks.get(0).isCorrupt() != true) { - try { - LOG.info("Waiting until block is marked as corrupt..."); - Thread.sleep(1000); - } catch (InterruptedException ie) { - } + // Now get block details and check if the block is corrupt blocks = dfsClient.getNamenode(). getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); + while (blocks.get(0).isCorrupt() != true) { + try { + LOG.info("Waiting until block is marked as corrupt..."); + Thread.sleep(1000); + } catch (InterruptedException ie) { + } + blocks = dfsClient.getNamenode(). + getBlockLocations(file1.toString(), 0, Long.MAX_VALUE); + } + replicaCount = blocks.get(0).getLocations().length; + assertTrue(replicaCount == 1); + } finally { + cluster.shutdown(); } - replicaCount = blocks.get(0).getLocations().length; - assertTrue(replicaCount == 1); - cluster.shutdown(); } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java index a02c88f9d11959..c194b848d48958 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java @@ -221,7 +221,7 @@ public void testRollingUpgradeWithQJM() throws Exception { final Path baz = new Path("/baz"); final RollingUpgradeInfo info1; - { + try { final DistributedFileSystem dfs = cluster.getFileSystem(); dfs.mkdirs(foo); @@ -235,6 +235,7 @@ public void testRollingUpgradeWithQJM() throws Exception { assertEquals(info1, dfs.rollingUpgrade(RollingUpgradeAction.QUERY)); dfs.mkdirs(bar); + } finally { cluster.shutdown(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java index 810f7e1864d174..905b83c9eabd38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderFactory.java @@ -133,19 +133,22 @@ public void testFallbackFromShortCircuitToUnixDomainTraffic() MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build(); - cluster.waitActive(); - FileSystem dfs = FileSystem.get(cluster.getURI(0), clientConf); - String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 8193; - final int SEED = 0xFADED; - DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE)); - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - assertTrue(Arrays.equals(contents, expected)); - cluster.shutdown(); - sockDir.close(); + try { + cluster.waitActive(); + FileSystem dfs = FileSystem.get(cluster.getURI(0), clientConf); + String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 8193; + final int SEED = 0xFADED; + DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE)); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + assertTrue(Arrays.equals(contents, expected)); + } finally { + cluster.shutdown(); + sockDir.close(); + } } /** @@ -210,42 +213,45 @@ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { "testMultipleWaitersOnShortCircuitCache", sockDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); - final String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 4000; - final int SEED = 0xFADED; - final int NUM_THREADS = 10; - DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - Runnable readerRunnable = new Runnable() { - @Override - public void run() { - try { - byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE)); - assertFalse(creationIsBlocked.get()); - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - assertTrue(Arrays.equals(contents, expected)); - } catch (Throwable e) { - LOG.error("readerRunnable error", e); - testFailed.set(true); + try { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4000; + final int SEED = 0xFADED; + final int NUM_THREADS = 10; + DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + Runnable readerRunnable = new Runnable() { + @Override + public void run() { + try { + byte contents[] = DFSTestUtil.readFileBuffer(dfs, new Path(TEST_FILE)); + assertFalse(creationIsBlocked.get()); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + assertTrue(Arrays.equals(contents, expected)); + } catch (Throwable e) { + LOG.error("readerRunnable error", e); + testFailed.set(true); + } } + }; + Thread threads[] = new Thread[NUM_THREADS]; + for (int i = 0; i < NUM_THREADS; i++) { + threads[i] = new Thread(readerRunnable); + threads[i].start(); } - }; - Thread threads[] = new Thread[NUM_THREADS]; - for (int i = 0; i < NUM_THREADS; i++) { - threads[i] = new Thread(readerRunnable); - threads[i].start(); - } - Thread.sleep(500); - latch.countDown(); - for (int i = 0; i < NUM_THREADS; i++) { - Uninterruptibles.joinUninterruptibly(threads[i]); + Thread.sleep(500); + latch.countDown(); + for (int i = 0; i < NUM_THREADS; i++) { + Uninterruptibles.joinUninterruptibly(threads[i]); + } + assertFalse(testFailed.get()); + } finally { + cluster.shutdown(); + sockDir.close(); } - cluster.shutdown(); - sockDir.close(); - assertFalse(testFailed.get()); } /** @@ -281,71 +287,74 @@ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { "testShortCircuitCacheTemporaryFailure", sockDir); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); - final String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 4000; - final int NUM_THREADS = 2; - final int SEED = 0xFADED; - final CountDownLatch gotFailureLatch = new CountDownLatch(NUM_THREADS); - final CountDownLatch shouldRetryLatch = new CountDownLatch(1); - DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - Runnable readerRunnable = new Runnable() { - @Override - public void run() { - try { - // First time should fail. - List locatedBlocks = - cluster.getNameNode().getRpcServer().getBlockLocations( - TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks(); - LocatedBlock lblock = locatedBlocks.get(0); // first block - BlockReader blockReader = null; + try { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4000; + final int NUM_THREADS = 2; + final int SEED = 0xFADED; + final CountDownLatch gotFailureLatch = new CountDownLatch(NUM_THREADS); + final CountDownLatch shouldRetryLatch = new CountDownLatch(1); + DFSTestUtil.createFile(dfs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + Runnable readerRunnable = new Runnable() { + @Override + public void run() { try { - blockReader = BlockReaderTestUtil.getBlockReader( - cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); - fail("expected getBlockReader to fail the first time."); - } catch (Throwable t) { - assertTrue(t.getMessage().contains("TCP reads were disabled for testing"), - "expected to see 'TCP reads were disabled " - + "for testing' in exception " + t); - } finally { - if (blockReader != null) blockReader.close(); // keep findbugs happy - } - gotFailureLatch.countDown(); - shouldRetryLatch.await(); + // First time should fail. + List locatedBlocks = + cluster.getNameNode().getRpcServer().getBlockLocations( + TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks(); + LocatedBlock lblock = locatedBlocks.get(0); // first block + BlockReader blockReader = null; + try { + blockReader = BlockReaderTestUtil.getBlockReader( + cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); + fail("expected getBlockReader to fail the first time."); + } catch (Throwable t) { + assertTrue(t.getMessage().contains("TCP reads were disabled for testing"), + "expected to see 'TCP reads were disabled " + + "for testing' in exception " + t); + } finally { + if (blockReader != null) blockReader.close(); // keep findbugs happy + } + gotFailureLatch.countDown(); + shouldRetryLatch.await(); - // Second time should succeed. - try { - blockReader = BlockReaderTestUtil.getBlockReader( - cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); + // Second time should succeed. + try { + blockReader = BlockReaderTestUtil.getBlockReader( + cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); + } catch (Throwable t) { + LOG.error("error trying to retrieve a block reader " + + "the second time.", t); + throw t; + } finally { + if (blockReader != null) blockReader.close(); + } } catch (Throwable t) { - LOG.error("error trying to retrieve a block reader " + - "the second time.", t); - throw t; - } finally { - if (blockReader != null) blockReader.close(); + LOG.error("getBlockReader failure", t); + testFailed.set(true); } - } catch (Throwable t) { - LOG.error("getBlockReader failure", t); - testFailed.set(true); } + }; + Thread threads[] = new Thread[NUM_THREADS]; + for (int i = 0; i < NUM_THREADS; i++) { + threads[i] = new Thread(readerRunnable); + threads[i].start(); } - }; - Thread threads[] = new Thread[NUM_THREADS]; - for (int i = 0; i < NUM_THREADS; i++) { - threads[i] = new Thread(readerRunnable); - threads[i].start(); - } - gotFailureLatch.await(); - replicaCreationShouldFail.set(false); - shouldRetryLatch.countDown(); - for (int i = 0; i < NUM_THREADS; i++) { - Uninterruptibles.joinUninterruptibly(threads[i]); + gotFailureLatch.await(); + replicaCreationShouldFail.set(false); + shouldRetryLatch.countDown(); + for (int i = 0; i < NUM_THREADS; i++) { + Uninterruptibles.joinUninterruptibly(threads[i]); + } + assertFalse(testFailed.get()); + } finally { + cluster.shutdown(); + sockDir.close(); } - cluster.shutdown(); - sockDir.close(); - assertFalse(testFailed.get()); } /** @@ -452,38 +461,41 @@ public void testShortCircuitReadFromServerWithoutShm() throws Exception { DFSInputStream.tcpReadsDisabledForTesting = true; final MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build(); - cluster.waitActive(); - clientConf.set(DFS_CLIENT_CONTEXT, - "testShortCircuitReadFromServerWithoutShm_clientContext"); - final DistributedFileSystem fs = - (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf); - final String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 4000; - final int SEED = 0xFADEC; - DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE)); - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - assertTrue(Arrays.equals(contents, expected)); - final ShortCircuitCache cache = - fs.getClient().getClientContext().getShortCircuitCache(0); - final DatanodeInfo datanode = new DatanodeInfoBuilder() - .setNodeID(cluster.getDataNodes().get(0).getDatanodeId()) - .build(); - cache.getDfsClientShmManager().visit(new Visitor() { - @Override - public void visit(HashMap info) - throws IOException { - assertEquals(1, info.size()); - PerDatanodeVisitorInfo vinfo = info.get(datanode); - assertTrue(vinfo.disabled); - assertEquals(0, vinfo.full.size()); - assertEquals(0, vinfo.notFull.size()); - } - }); - cluster.shutdown(); - sockDir.close(); + try { + cluster.waitActive(); + clientConf.set(DFS_CLIENT_CONTEXT, + "testShortCircuitReadFromServerWithoutShm_clientContext"); + final DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4000; + final int SEED = 0xFADEC; + DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE)); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + assertTrue(Arrays.equals(contents, expected)); + final ShortCircuitCache cache = + fs.getClient().getClientContext().getShortCircuitCache(0); + final DatanodeInfo datanode = new DatanodeInfoBuilder() + .setNodeID(cluster.getDataNodes().get(0).getDatanodeId()) + .build(); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + assertEquals(1, info.size()); + PerDatanodeVisitorInfo vinfo = info.get(datanode); + assertTrue(vinfo.disabled); + assertEquals(0, vinfo.full.size()); + assertEquals(0, vinfo.notFull.size()); + } + }); + } finally { + cluster.shutdown(); + sockDir.close(); + } } /** @@ -499,27 +511,30 @@ public void testShortCircuitReadFromClientWithoutShm() throws Exception { DFSInputStream.tcpReadsDisabledForTesting = true; final MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build(); - cluster.waitActive(); - clientConf.setInt( - DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS, 0); - clientConf.set(DFS_CLIENT_CONTEXT, - "testShortCircuitReadFromClientWithoutShm_clientContext"); - final DistributedFileSystem fs = - (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf); - final String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 4000; - final int SEED = 0xFADEC; - DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE)); - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - assertTrue(Arrays.equals(contents, expected)); - final ShortCircuitCache cache = - fs.getClient().getClientContext().getShortCircuitCache(0); - assertEquals(null, cache.getDfsClientShmManager()); - cluster.shutdown(); - sockDir.close(); + try { + cluster.waitActive(); + clientConf.setInt( + DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS, 0); + clientConf.set(DFS_CLIENT_CONTEXT, + "testShortCircuitReadFromClientWithoutShm_clientContext"); + final DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(0), clientConf); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4000; + final int SEED = 0xFADEC; + DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE)); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + assertTrue(Arrays.equals(contents, expected)); + final ShortCircuitCache cache = + fs.getClient().getClientContext().getShortCircuitCache(0); + assertEquals(null, cache.getDfsClientShmManager()); + } finally { + cluster.shutdown(); + sockDir.close(); + } } /** @@ -535,25 +550,28 @@ public void testShortCircuitCacheShutdown() throws Exception { DFSInputStream.tcpReadsDisabledForTesting = true; final MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build(); - cluster.waitActive(); - final DistributedFileSystem fs = - (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf); - final String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 4000; - final int SEED = 0xFADEC; - DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE)); - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - assertTrue(Arrays.equals(contents, expected)); - final ShortCircuitCache cache = - fs.getClient().getClientContext().getShortCircuitCache(0); - cache.close(); - assertTrue(cache.getDfsClientShmManager(). - getDomainSocketWatcher().isClosed()); - cluster.shutdown(); - sockDir.close(); + try { + cluster.waitActive(); + final DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4000; + final int SEED = 0xFADEC; + DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + byte contents[] = DFSTestUtil.readFileBuffer(fs, new Path(TEST_FILE)); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + assertTrue(Arrays.equals(contents, expected)); + final ShortCircuitCache cache = + fs.getClient().getClientContext().getShortCircuitCache(0); + cache.close(); + assertTrue(cache.getDfsClientShmManager(). + getDomainSocketWatcher().isClosed()); + } finally { + cluster.shutdown(); + sockDir.close(); + } } /** @@ -593,84 +611,87 @@ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { "testPurgingClosedReplicas", sockDir); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - final DistributedFileSystem dfs = cluster.getFileSystem(); - final String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 4095; - final int SEED = 0xFADE0; - final DistributedFileSystem fs = - (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf); - DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - - final Semaphore sem = new Semaphore(0); - final List locatedBlocks = - cluster.getNameNode().getRpcServer().getBlockLocations( - TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks(); - final LocatedBlock lblock = locatedBlocks.get(0); // first block - final byte[] buf = new byte[TEST_FILE_LEN]; - Runnable readerRunnable = new Runnable() { - @Override - public void run() { - try { - while (true) { - BlockReader blockReader = null; - try { - blockReader = BlockReaderTestUtil.getBlockReader( - cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); - sem.release(); + try { + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 4095; + final int SEED = 0xFADE0; + final DistributedFileSystem fs = + (DistributedFileSystem)FileSystem.get(cluster.getURI(0), conf); + DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + + final Semaphore sem = new Semaphore(0); + final List locatedBlocks = + cluster.getNameNode().getRpcServer().getBlockLocations( + TEST_FILE, 0, TEST_FILE_LEN).getLocatedBlocks(); + final LocatedBlock lblock = locatedBlocks.get(0); // first block + final byte[] buf = new byte[TEST_FILE_LEN]; + Runnable readerRunnable = new Runnable() { + @Override + public void run() { + try { + while (true) { + BlockReader blockReader = null; try { - blockReader.readAll(buf, 0, TEST_FILE_LEN); + blockReader = BlockReaderTestUtil.getBlockReader( + cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); + sem.release(); + try { + blockReader.readAll(buf, 0, TEST_FILE_LEN); + } finally { + sem.acquireUninterruptibly(); + } + } catch (ClosedByInterruptException e) { + LOG.info("got the expected ClosedByInterruptException", e); + sem.release(); + break; } finally { - sem.acquireUninterruptibly(); + if (blockReader != null) blockReader.close(); } - } catch (ClosedByInterruptException e) { - LOG.info("got the expected ClosedByInterruptException", e); - sem.release(); - break; - } finally { - if (blockReader != null) blockReader.close(); + LOG.info("read another " + TEST_FILE_LEN + " bytes."); } - LOG.info("read another " + TEST_FILE_LEN + " bytes."); + } catch (Throwable t) { + LOG.error("getBlockReader failure", t); + testFailed.set(true); + sem.release(); } - } catch (Throwable t) { - LOG.error("getBlockReader failure", t); - testFailed.set(true); - sem.release(); } + }; + Thread thread = new Thread(readerRunnable); + thread.start(); + + // While the thread is reading, send it interrupts. + // These should trigger a ClosedChannelException. + while (thread.isAlive()) { + sem.acquireUninterruptibly(); + thread.interrupt(); + sem.release(); } - }; - Thread thread = new Thread(readerRunnable); - thread.start(); - - // While the thread is reading, send it interrupts. - // These should trigger a ClosedChannelException. - while (thread.isAlive()) { - sem.acquireUninterruptibly(); - thread.interrupt(); - sem.release(); - } - assertFalse(testFailed.get()); + assertFalse(testFailed.get()); + + // We should be able to read from the file without + // getting a ClosedChannelException. + BlockReader blockReader = null; + try { + blockReader = BlockReaderTestUtil.getBlockReader( + cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); + blockReader.readFully(buf, 0, TEST_FILE_LEN); + } finally { + if (blockReader != null) blockReader.close(); + } + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + assertTrue(Arrays.equals(buf, expected)); - // We should be able to read from the file without - // getting a ClosedChannelException. - BlockReader blockReader = null; - try { - blockReader = BlockReaderTestUtil.getBlockReader( - cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN); - blockReader.readFully(buf, 0, TEST_FILE_LEN); + // Another ShortCircuitReplica object should have been created. + assertEquals(2, replicasCreated.get()); + + dfs.close(); } finally { - if (blockReader != null) blockReader.close(); + cluster.shutdown(); + sockDir.close(); } - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - assertTrue(Arrays.equals(buf, expected)); - - // Another ShortCircuitReplica object should have been created. - assertEquals(2, replicasCreated.get()); - - dfs.close(); - cluster.shutdown(); - sockDir.close(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java index 88a1f8be3b2585..dc531fbc1f9774 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/TestBlockReaderLocalLegacy.java @@ -98,45 +98,48 @@ public void testStablePositionAfterCorruptRead() throws Exception { File basedir = new File(GenericTestUtils.getRandomizedTempPath()); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, basedir).numDataNodes(1).build(); - cluster.waitActive(); - FileSystem fs = cluster.getFileSystem(); - - Path path = new Path("/corrupted"); - - DFSTestUtil.createFile(fs, path, FILE_LENGTH, REPL_FACTOR, 12345L); - DFSTestUtil.waitReplication(fs, path, REPL_FACTOR); - - ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, path); - int blockFilesCorrupted = cluster.corruptBlockOnDataNodes(block); - assertEquals(REPL_FACTOR, blockFilesCorrupted, "All replicas not corrupted"); - - FSDataInputStream dis = cluster.getFileSystem().open(path); - ByteBuffer buf = ByteBuffer.allocateDirect((int)FILE_LENGTH); - boolean sawException = false; - try { - dis.read(buf); - } catch (ChecksumException ex) { - sawException = true; - } - - assertTrue(sawException); - assertEquals(0, buf.position()); - assertEquals(buf.capacity(), buf.limit()); - - dis = cluster.getFileSystem().open(path); - buf.position(3); - buf.limit(25); - sawException = false; try { - dis.read(buf); - } catch (ChecksumException ex) { - sawException = true; + cluster.waitActive(); + FileSystem fs = cluster.getFileSystem(); + + Path path = new Path("/corrupted"); + + DFSTestUtil.createFile(fs, path, FILE_LENGTH, REPL_FACTOR, 12345L); + DFSTestUtil.waitReplication(fs, path, REPL_FACTOR); + + ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, path); + int blockFilesCorrupted = cluster.corruptBlockOnDataNodes(block); + assertEquals(REPL_FACTOR, blockFilesCorrupted, "All replicas not corrupted"); + + FSDataInputStream dis = cluster.getFileSystem().open(path); + ByteBuffer buf = ByteBuffer.allocateDirect((int)FILE_LENGTH); + boolean sawException = false; + try { + dis.read(buf); + } catch (ChecksumException ex) { + sawException = true; + } + + assertTrue(sawException); + assertEquals(0, buf.position()); + assertEquals(buf.capacity(), buf.limit()); + + dis = cluster.getFileSystem().open(path); + buf.position(3); + buf.limit(25); + sawException = false; + try { + dis.read(buf); + } catch (ChecksumException ex) { + sawException = true; + } + + assertTrue(sawException); + assertEquals(3, buf.position()); + assertEquals(25, buf.limit()); + } finally { + cluster.shutdown(); } - - assertTrue(sawException); - assertEquals(3, buf.position()); - assertEquals(25, buf.limit()); - cluster.shutdown(); } @Test @@ -148,26 +151,29 @@ public void testBothOldAndNewShortCircuitConfigured() throws Exception { HdfsConfiguration conf = getConfiguration(socketDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - socketDir.close(); - FileSystem fs = cluster.getFileSystem(); - - Path path = new Path("/foo"); - byte orig[] = new byte[FILE_LENGTH]; - for (int i = 0; i < orig.length; i++) { - orig[i] = (byte)(i%10); + try { + cluster.waitActive(); + socketDir.close(); + FileSystem fs = cluster.getFileSystem(); + + Path path = new Path("/foo"); + byte orig[] = new byte[FILE_LENGTH]; + for (int i = 0; i < orig.length; i++) { + orig[i] = (byte)(i%10); + } + FSDataOutputStream fos = fs.create(path, (short)1); + fos.write(orig); + fos.close(); + DFSTestUtil.waitReplication(fs, path, REPL_FACTOR); + FSDataInputStream fis = cluster.getFileSystem().open(path); + byte buf[] = new byte[FILE_LENGTH]; + IOUtils.readFully(fis, buf, 0, FILE_LENGTH); + fis.close(); + assertArrayEquals(orig, buf); + Arrays.equals(orig, buf); + } finally { + cluster.shutdown(); } - FSDataOutputStream fos = fs.create(path, (short)1); - fos.write(orig); - fos.close(); - DFSTestUtil.waitReplication(fs, path, REPL_FACTOR); - FSDataInputStream fis = cluster.getFileSystem().open(path); - byte buf[] = new byte[FILE_LENGTH]; - IOUtils.readFully(fis, buf, 0, FILE_LENGTH); - fis.close(); - assertArrayEquals(orig, buf); - Arrays.equals(orig, buf); - cluster.shutdown(); } @Test @@ -180,53 +186,56 @@ public void testBlockReaderLocalLegacyWithAppend() throws Exception { File basedir = new File(GenericTestUtils.getRandomizedTempPath()); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, basedir).numDataNodes(1).build(); - cluster.waitActive(); - - final DistributedFileSystem dfs = cluster.getFileSystem(); - final Path path = new Path("/testBlockReaderLocalLegacy"); - DFSTestUtil.createFile(dfs, path, 10, REPL_FACTOR, 0); - DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR); - - final ClientDatanodeProtocol proxy; - final Token token; - final ExtendedBlock originalBlock; - final long originalGS; - { - final LocatedBlock lb = cluster.getNameNode().getRpcServer() - .getBlockLocations(path.toString(), 0, 1).get(0); - proxy = DFSUtilClient.createClientDatanodeProtocolProxy( - lb.getLocations()[0], conf, 60000, false); - token = lb.getBlockToken(); - - // get block and generation stamp - final ExtendedBlock blk = new ExtendedBlock(lb.getBlock()); - originalBlock = new ExtendedBlock(blk); - originalGS = originalBlock.getGenerationStamp(); - - // test getBlockLocalPathInfo - final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(blk, token); - assertEquals(originalGS, info.getBlock().getGenerationStamp()); - } - - { // append one byte - FSDataOutputStream out = dfs.append(path); - out.write(1); - out.close(); - } - - { - // get new generation stamp - final LocatedBlock lb = cluster.getNameNode().getRpcServer() - .getBlockLocations(path.toString(), 0, 1).get(0); - final long newGS = lb.getBlock().getGenerationStamp(); - assertTrue(newGS > originalGS); - - // getBlockLocalPathInfo using the original block. - assertEquals(originalGS, originalBlock.getGenerationStamp()); - final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo( - originalBlock, token); - assertEquals(newGS, info.getBlock().getGenerationStamp()); + try { + cluster.waitActive(); + + final DistributedFileSystem dfs = cluster.getFileSystem(); + final Path path = new Path("/testBlockReaderLocalLegacy"); + DFSTestUtil.createFile(dfs, path, 10, REPL_FACTOR, 0); + DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR); + + final ClientDatanodeProtocol proxy; + final Token token; + final ExtendedBlock originalBlock; + final long originalGS; + { + final LocatedBlock lb = cluster.getNameNode().getRpcServer() + .getBlockLocations(path.toString(), 0, 1).get(0); + proxy = DFSUtilClient.createClientDatanodeProtocolProxy( + lb.getLocations()[0], conf, 60000, false); + token = lb.getBlockToken(); + + // get block and generation stamp + final ExtendedBlock blk = new ExtendedBlock(lb.getBlock()); + originalBlock = new ExtendedBlock(blk); + originalGS = originalBlock.getGenerationStamp(); + + // test getBlockLocalPathInfo + final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(blk, token); + assertEquals(originalGS, info.getBlock().getGenerationStamp()); + } + + { // append one byte + FSDataOutputStream out = dfs.append(path); + out.write(1); + out.close(); + } + + { + // get new generation stamp + final LocatedBlock lb = cluster.getNameNode().getRpcServer() + .getBlockLocations(path.toString(), 0, 1).get(0); + final long newGS = lb.getBlock().getGenerationStamp(); + assertTrue(newGS > originalGS); + + // getBlockLocalPathInfo using the original block. + assertEquals(originalGS, originalBlock.getGenerationStamp()); + final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo( + originalBlock, token); + assertEquals(newGS, info.getBlock().getGenerationStamp()); + } + } finally { + cluster.shutdown(); } - cluster.shutdown(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java index 298c856655dda6..8f1c0de829f15b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java @@ -36,12 +36,16 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; public class TestNNWithQJM { + private static final Logger LOG = + LoggerFactory.getLogger(TestNNWithQJM.class); final Configuration conf = new HdfsConfiguration(); private MiniJournalCluster mjc = null; private final Path TEST_PATH = new Path("/test-dir"); @@ -168,7 +172,19 @@ public void testNewNamenodeTakesOverWriter() throws Exception { "Could not sync enough journals to persistent storage", re); } } finally { - //cluster.shutdown(); + if (cluster != null) { + try { + cluster.shutdown(); + } catch (ExitUtil.ExitException e) { + // Expected: this NN was fenced by the second cluster above, so + // closing its edit log cannot reach a journal quorum and the + // shutdown terminates with "Could not sync enough journals". + // Releasing what can be released still beats leaving the NN + // running for the remaining tests in this class. + LOG.warn("Expected exit while shutting down the fenced NN", e); + ExitUtil.resetFirstExitException(); + } + } } } @@ -188,23 +204,34 @@ public void testMismatchedNNIsRejected() throws Exception { .manageNameDfsDirs(false) .build(); cluster.shutdown(); - - // Reformat just the on-disk portion - Configuration onDiskOnly = new Configuration(conf); - onDiskOnly.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, defaultEditsDir); - NameNode.format(onDiskOnly); + // Null out the reference: the build below is expected to throw before + // reassigning it, and the finally must not shut this cluster down a + // second time -- that would re-run the ExitUtil exit check, whose + // AssertionError could mask the test's real failure. + cluster = null; - // Start the NN - should fail because the JNs are still formatted - // with the old namespace ID. try { - ExitUtil.disableSystemExit(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) - .manageNameDfsDirs(false).format(false).checkExitOnShutdown(false) - .build(); - fail("New NN with different namespace should have been rejected"); - } catch (IOException ioe) { - GenericTestUtils.assertExceptionContains( - "recoverUnfinalizedSegments failed for too many journals", ioe); + // Reformat just the on-disk portion + Configuration onDiskOnly = new Configuration(conf); + onDiskOnly.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, defaultEditsDir); + NameNode.format(onDiskOnly); + + // Start the NN - should fail because the JNs are still formatted + // with the old namespace ID. + try { + ExitUtil.disableSystemExit(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) + .manageNameDfsDirs(false).format(false).checkExitOnShutdown(false) + .build(); + fail("New NN with different namespace should have been rejected"); + } catch (IOException ioe) { + GenericTestUtils.assertExceptionContains( + "recoverUnfinalizedSegments failed for too many journals", ioe); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java index 95f97d1a93eb3f..9c0071d2d0eef2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockReportRateLimiting.java @@ -138,28 +138,31 @@ public void removeBlockReportLease(DatanodeDescriptor node, long leaseId) { final int NUM_DATANODES = 5; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build(); - cluster.waitActive(); - for (int n = 1; n <= NUM_DATANODES; n++) { - LOG.info("Waiting for " + n + " datanode(s) to report in."); - fbrSem.release(); - Uninterruptibles.sleepUninterruptibly(20, TimeUnit.MILLISECONDS); - final int currentN = n; - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - synchronized (injector) { - if (fbrDns.size() > currentN) { - setFailure(failure, "Expected at most " + currentN + - " datanodes to have sent a block report, but actually " + - fbrDns.size() + " have."); + try { + cluster.waitActive(); + for (int n = 1; n <= NUM_DATANODES; n++) { + LOG.info("Waiting for " + n + " datanode(s) to report in."); + fbrSem.release(); + Uninterruptibles.sleepUninterruptibly(20, TimeUnit.MILLISECONDS); + final int currentN = n; + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + synchronized (injector) { + if (fbrDns.size() > currentN) { + setFailure(failure, "Expected at most " + currentN + + " datanodes to have sent a block report, but actually " + + fbrDns.size() + " have."); + } + return (fbrDns.size() >= currentN); } - return (fbrDns.size() >= currentN); } - } - }, 25, 50000); + }, 25, 50000); + } + assertEquals("", failure.get()); + } finally { + cluster.shutdown(); } - cluster.shutdown(); - assertEquals("", failure.get()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java index 7e7a8b6128a3f2..0c736d012e2ede 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestStorageBlockPoolUsageStdDev.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.eclipse.jetty.util.ajax.JSON; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -73,6 +74,14 @@ public void setup() throws Exception { fs = cluster.getFileSystem(); } + @AfterEach + public void tearDown() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + /** * Create files of different sizes for each datanode. * Ensure that the file size is smaller than the blocksize diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java index ececeb07e81419..8d8d8bffe12398 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java @@ -58,71 +58,74 @@ public void testBlockCounters() throws Exception { final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - final DatanodeProtocolClientSideTranslatorPB spy = - InternalDataNodeTestUtils.spyOnBposToNN( - cluster.getDataNodes().get(0), cluster.getNameNode()); - final DataNode datanode = cluster.getDataNodes().get(0); - - /* We should get 0 incremental block report. */ - Mockito.verify(spy, timeout(60000).times(0)).blockReceivedAndDeleted( - any(DatanodeRegistration.class), - anyString(), - any(StorageReceivedDeletedBlocks[].class)); - - /* - * Create fake blocks notification on the DataNode. This will be sent with - * the next incremental block report. - */ - final BPServiceActor actor = - datanode.getAllBpOs().get(0).getBPServiceActors().get(0); - final FsDatasetSpi dataset = datanode.getFSDataset(); - final DatanodeStorage storage; - try (FsDatasetSpi.FsVolumeReferences volumes = - dataset.getFsVolumeReferences()) { - storage = dataset.getStorage(volumes.get(0).getStorageID()); + try { + cluster.waitActive(); + final DatanodeProtocolClientSideTranslatorPB spy = + InternalDataNodeTestUtils.spyOnBposToNN( + cluster.getDataNodes().get(0), cluster.getNameNode()); + final DataNode datanode = cluster.getDataNodes().get(0); + + /* We should get 0 incremental block report. */ + Mockito.verify(spy, timeout(60000).times(0)).blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); + + /* + * Create fake blocks notification on the DataNode. This will be sent with + * the next incremental block report. + */ + final BPServiceActor actor = + datanode.getAllBpOs().get(0).getBPServiceActors().get(0); + final FsDatasetSpi dataset = datanode.getFSDataset(); + final DatanodeStorage storage; + try (FsDatasetSpi.FsVolumeReferences volumes = + dataset.getFsVolumeReferences()) { + storage = dataset.getStorage(volumes.get(0).getStorageID()); + } + + ReceivedDeletedBlockInfo rdbi = null; + /* block at status of RECEIVING_BLOCK */ + rdbi = new ReceivedDeletedBlockInfo( + new Block(5678, 512, 1000), BlockStatus.RECEIVING_BLOCK, null); + actor.getIbrManager().addRDBI(rdbi, storage); + + /* block at status of RECEIVED_BLOCK */ + rdbi = new ReceivedDeletedBlockInfo( + new Block(5679, 512, 1000), BlockStatus.RECEIVED_BLOCK, null); + actor.getIbrManager().addRDBI(rdbi, storage); + + /* block at status of DELETED_BLOCK */ + rdbi = new ReceivedDeletedBlockInfo( + new Block(5680, 512, 1000), BlockStatus.DELETED_BLOCK, null); + actor.getIbrManager().addRDBI(rdbi, storage); + + /* verify counters before sending IBR */ + verifyBlockCounters(datanode, 3, 1, 1, 1); + + /* Manually trigger a block report. */ + datanode.triggerBlockReport( + new BlockReportOptions.Factory(). + setIncremental(true). + build() + ); + + /* + * triggerBlockReport returns before the block report is actually sent. Wait + * for it to be sent here. + */ + Mockito.verify(spy, timeout(60000).times(1)). + blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); + + /* verify counters after sending IBR */ + verifyBlockCounters(datanode, 0, 0, 0, 0); + + } finally { + cluster.shutdown(); } - - ReceivedDeletedBlockInfo rdbi = null; - /* block at status of RECEIVING_BLOCK */ - rdbi = new ReceivedDeletedBlockInfo( - new Block(5678, 512, 1000), BlockStatus.RECEIVING_BLOCK, null); - actor.getIbrManager().addRDBI(rdbi, storage); - - /* block at status of RECEIVED_BLOCK */ - rdbi = new ReceivedDeletedBlockInfo( - new Block(5679, 512, 1000), BlockStatus.RECEIVED_BLOCK, null); - actor.getIbrManager().addRDBI(rdbi, storage); - - /* block at status of DELETED_BLOCK */ - rdbi = new ReceivedDeletedBlockInfo( - new Block(5680, 512, 1000), BlockStatus.DELETED_BLOCK, null); - actor.getIbrManager().addRDBI(rdbi, storage); - - /* verify counters before sending IBR */ - verifyBlockCounters(datanode, 3, 1, 1, 1); - - /* Manually trigger a block report. */ - datanode.triggerBlockReport( - new BlockReportOptions.Factory(). - setIncremental(true). - build() - ); - - /* - * triggerBlockReport returns before the block report is actually sent. Wait - * for it to be sent here. - */ - Mockito.verify(spy, timeout(60000).times(1)). - blockReceivedAndDeleted( - any(DatanodeRegistration.class), - anyString(), - any(StorageReceivedDeletedBlocks[].class)); - - /* verify counters after sending IBR */ - verifyBlockCounters(datanode, 0, 0, 0, 0); - - cluster.shutdown(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java index ff403c65310544..1e261c6a02ba5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeInitStorage.java @@ -78,9 +78,9 @@ public void testDataNodeInitStorage() throws Throwable { // Start a cluster so that SimulatedFsDatasetVerifier constructor is // invoked. - MiniDFSCluster cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - cluster.shutdown(); + try (MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build()) { + cluster.waitActive(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java index bf72ea9ede52c2..4329644e10c24f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java @@ -606,11 +606,12 @@ public void testNNRpcMetricsWithNonHA() throws IOException { // heartbeat periodically to NN during running test case, and bpServiceActor // only sends heartbeat once after startup conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.waitActive(); - DataNode dn = cluster.getDataNodes().get(0); - MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); - assertCounter("HeartbeatsNumOps", 1L, rb); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) { + cluster.waitActive(); + DataNode dn = cluster.getDataNodes().get(0); + MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); + assertCounter("HeartbeatsNumOps", 1L, rb); + } } @Test @@ -676,15 +677,16 @@ public void testNNRpcMetricsWithHA() throws IOException { // heartbeat periodically to NN during running test case, and bpServiceActor // only sends heartbeat once after startup conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( - MiniDFSNNTopology.simpleHATopology()).build(); - cluster.waitActive(); - DataNode dn = cluster.getDataNodes().get(0); - cluster.transitionToActive(0); - MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); - assertCounter("HeartbeatsForminidfs-ns-nn1NumOps", 1L, rb); - assertCounter("HeartbeatsForminidfs-ns-nn2NumOps", 1L, rb); - assertCounter("HeartbeatsNumOps", 2L, rb); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( + MiniDFSNNTopology.simpleHATopology()).build()) { + cluster.waitActive(); + DataNode dn = cluster.getDataNodes().get(0); + cluster.transitionToActive(0); + MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); + assertCounter("HeartbeatsForminidfs-ns-nn1NumOps", 1L, rb); + assertCounter("HeartbeatsForminidfs-ns-nn2NumOps", 1L, rb); + assertCounter("HeartbeatsNumOps", 2L, rb); + } } @Test @@ -694,14 +696,15 @@ public void testNNRpcMetricsWithFederation() throws IOException { // heartbeat periodically to NN during running test case, and bpServiceActor // only sends heartbeat once after startup conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( - MiniDFSNNTopology.simpleFederatedTopology("ns1,ns2")).build(); - cluster.waitActive(); - DataNode dn = cluster.getDataNodes().get(0); - MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); - assertCounter("HeartbeatsForns1NumOps", 1L, rb); - assertCounter("HeartbeatsForns2NumOps", 1L, rb); - assertCounter("HeartbeatsNumOps", 2L, rb); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( + MiniDFSNNTopology.simpleFederatedTopology("ns1,ns2")).build()) { + cluster.waitActive(); + DataNode dn = cluster.getDataNodes().get(0); + MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); + assertCounter("HeartbeatsForns1NumOps", 1L, rb); + assertCounter("HeartbeatsForns2NumOps", 1L, rb); + assertCounter("HeartbeatsNumOps", 2L, rb); + } } @Test @@ -711,17 +714,18 @@ public void testNNRpcMetricsWithFederationAndHA() throws IOException { // heartbeat periodically to NN during running test case, and bpServiceActor // only sends heartbeat once after startup conf.setTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY, 1, TimeUnit.HOURS); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( - MiniDFSNNTopology.simpleHAFederatedTopology(2)).build(); - cluster.waitActive(); - DataNode dn = cluster.getDataNodes().get(0); - MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); - - assertCounter("HeartbeatsForns0-nn0NumOps", 1L, rb); - assertCounter("HeartbeatsForns0-nn1NumOps", 1L, rb); - assertCounter("HeartbeatsForns1-nn0NumOps", 1L, rb); - assertCounter("HeartbeatsForns1-nn1NumOps", 1L, rb); - assertCounter("HeartbeatsNumOps", 4L, rb); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( + MiniDFSNNTopology.simpleHAFederatedTopology(2)).build()) { + cluster.waitActive(); + DataNode dn = cluster.getDataNodes().get(0); + MetricsRecordBuilder rb = getMetrics(dn.getMetrics().name()); + + assertCounter("HeartbeatsForns0-nn0NumOps", 1L, rb); + assertCounter("HeartbeatsForns0-nn1NumOps", 1L, rb); + assertCounter("HeartbeatsForns1-nn0NumOps", 1L, rb); + assertCounter("HeartbeatsForns1-nn1NumOps", 1L, rb); + assertCounter("HeartbeatsNumOps", 4L, rb); + } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java index 46dd31c31e2f11..480c0c5885bbcc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestHSync.java @@ -64,56 +64,59 @@ public void testHSyncWithAppend() throws Exception { private void testHSyncOperation(boolean testWithAppend) throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - final DistributedFileSystem fs = cluster.getFileSystem(); + try { + final DistributedFileSystem fs = cluster.getFileSystem(); - final Path p = new Path("/testHSync/foo"); - final int len = 1 << 16; - FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), - 4096, (short) 1, len, null); - if (testWithAppend) { - // re-open the file with append call + final Path p = new Path("/testHSync/foo"); + final int len = 1 << 16; + FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), + 4096, (short) 1, len, null); + if (testWithAppend) { + // re-open the file with append call + out.close(); + out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.SYNC_BLOCK), + 4096, null); + } + out.hflush(); + // hflush does not sync + checkSyncMetric(cluster, 0); + out.hsync(); + // hsync on empty file does nothing + checkSyncMetric(cluster, 0); + out.write(1); + checkSyncMetric(cluster, 0); + out.hsync(); + checkSyncMetric(cluster, 1); + // avoiding repeated hsyncs is a potential future optimization + out.hsync(); + checkSyncMetric(cluster, 2); + out.hflush(); + // hflush still does not sync + checkSyncMetric(cluster, 2); out.close(); - out = fs.append(p, EnumSet.of(CreateFlag.APPEND, CreateFlag.SYNC_BLOCK), - 4096, null); - } - out.hflush(); - // hflush does not sync - checkSyncMetric(cluster, 0); - out.hsync(); - // hsync on empty file does nothing - checkSyncMetric(cluster, 0); - out.write(1); - checkSyncMetric(cluster, 0); - out.hsync(); - checkSyncMetric(cluster, 1); - // avoiding repeated hsyncs is a potential future optimization - out.hsync(); - checkSyncMetric(cluster, 2); - out.hflush(); - // hflush still does not sync - checkSyncMetric(cluster, 2); - out.close(); - // close is sync'ing - checkSyncMetric(cluster, 3); + // close is sync'ing + checkSyncMetric(cluster, 3); - // same with a file created with out SYNC_BLOCK - out = fs.create(p, FsPermission.getDefault(), - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), - 4096, (short) 1, len, null); - out.hsync(); - checkSyncMetric(cluster, 3); - out.write(1); - checkSyncMetric(cluster, 3); - out.hsync(); - checkSyncMetric(cluster, 4); - // repeated hsyncs - out.hsync(); - checkSyncMetric(cluster, 5); - out.close(); - // close does not sync (not opened with SYNC_BLOCK) - checkSyncMetric(cluster, 5); - cluster.shutdown(); + // same with a file created with out SYNC_BLOCK + out = fs.create(p, FsPermission.getDefault(), + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), + 4096, (short) 1, len, null); + out.hsync(); + checkSyncMetric(cluster, 3); + out.write(1); + checkSyncMetric(cluster, 3); + out.hsync(); + checkSyncMetric(cluster, 4); + // repeated hsyncs + out.hsync(); + checkSyncMetric(cluster, 5); + out.close(); + // close does not sync (not opened with SYNC_BLOCK) + checkSyncMetric(cluster, 5); + } finally { + cluster.shutdown(); + } } /** Test hsync on an exact block boundary */ @@ -121,29 +124,32 @@ private void testHSyncOperation(boolean testWithAppend) throws IOException { public void testHSyncBlockBoundary() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - final FileSystem fs = cluster.getFileSystem(); + try { + final FileSystem fs = cluster.getFileSystem(); - final Path p = new Path("/testHSyncBlockBoundary/foo"); - final int len = 1 << 16; - final byte[] fileContents = AppendTestUtil.initBuffer(len); - FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), - 4096, (short) 1, len, null); - // fill exactly one block (tests the SYNC_BLOCK case) and flush - out.write(fileContents, 0, len); - out.hflush(); - // the full block should have caused a sync - checkSyncMetric(cluster, 1); - out.hsync(); - // first on block again - checkSyncMetric(cluster, 1); - // write one more byte and sync again - out.write(1); - out.hsync(); - checkSyncMetric(cluster, 2); - out.close(); - checkSyncMetric(cluster, 3); - cluster.shutdown(); + final Path p = new Path("/testHSyncBlockBoundary/foo"); + final int len = 1 << 16; + final byte[] fileContents = AppendTestUtil.initBuffer(len); + FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), + 4096, (short) 1, len, null); + // fill exactly one block (tests the SYNC_BLOCK case) and flush + out.write(fileContents, 0, len); + out.hflush(); + // the full block should have caused a sync + checkSyncMetric(cluster, 1); + out.hsync(); + // first on block again + checkSyncMetric(cluster, 1); + // write one more byte and sync again + out.write(1); + out.hsync(); + checkSyncMetric(cluster, 2); + out.close(); + checkSyncMetric(cluster, 3); + } finally { + cluster.shutdown(); + } } /** Test hsync via SequenceFiles */ @@ -151,33 +157,36 @@ public void testHSyncBlockBoundary() throws Exception { public void testSequenceFileSync() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + try { - final FileSystem fs = cluster.getFileSystem(); - final Path p = new Path("/testSequenceFileSync/foo"); - final int len = 1 << 16; - FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), - 4096, (short) 1, len, null); - Writer w = SequenceFile.createWriter(new Configuration(), - Writer.stream(out), - Writer.keyClass(RandomDatum.class), - Writer.valueClass(RandomDatum.class), - Writer.compression(CompressionType.NONE, new DefaultCodec())); - w.hflush(); - checkSyncMetric(cluster, 0); - w.hsync(); - checkSyncMetric(cluster, 1); - int seed = new Random().nextInt(); - RandomDatum.Generator generator = new RandomDatum.Generator(seed); - generator.next(); - w.append(generator.getKey(), generator.getValue()); - w.hsync(); - checkSyncMetric(cluster, 2); - w.close(); - checkSyncMetric(cluster, 2); - out.close(); - checkSyncMetric(cluster, 3); - cluster.shutdown(); + final FileSystem fs = cluster.getFileSystem(); + final Path p = new Path("/testSequenceFileSync/foo"); + final int len = 1 << 16; + FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), + 4096, (short) 1, len, null); + Writer w = SequenceFile.createWriter(new Configuration(), + Writer.stream(out), + Writer.keyClass(RandomDatum.class), + Writer.valueClass(RandomDatum.class), + Writer.compression(CompressionType.NONE, new DefaultCodec())); + w.hflush(); + checkSyncMetric(cluster, 0); + w.hsync(); + checkSyncMetric(cluster, 1); + int seed = new Random().nextInt(); + RandomDatum.Generator generator = new RandomDatum.Generator(seed); + generator.next(); + w.append(generator.getKey(), generator.getValue()); + w.hsync(); + checkSyncMetric(cluster, 2); + w.close(); + checkSyncMetric(cluster, 2); + out.close(); + checkSyncMetric(cluster, 3); + } finally { + cluster.shutdown(); + } } /** Test that syncBlock is correctly performed at replicas */ @@ -185,26 +194,29 @@ public void testSequenceFileSync() throws Exception { public void testHSyncWithReplication() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); - final FileSystem fs = cluster.getFileSystem(); + try { + final FileSystem fs = cluster.getFileSystem(); - final Path p = new Path("/testHSyncWithReplication/foo"); - final int len = 1 << 16; - FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), - 4096, (short) 3, len, null); - out.write(1); - out.hflush(); - checkSyncMetric(cluster, 0, 0); - checkSyncMetric(cluster, 1, 0); - checkSyncMetric(cluster, 2, 0); - out.hsync(); - checkSyncMetric(cluster, 0, 1); - checkSyncMetric(cluster, 1, 1); - checkSyncMetric(cluster, 2, 1); - out.hsync(); - checkSyncMetric(cluster, 0, 2); - checkSyncMetric(cluster, 1, 2); - checkSyncMetric(cluster, 2, 2); - cluster.shutdown(); + final Path p = new Path("/testHSyncWithReplication/foo"); + final int len = 1 << 16; + FSDataOutputStream out = fs.create(p, FsPermission.getDefault(), + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE, CreateFlag.SYNC_BLOCK), + 4096, (short) 3, len, null); + out.write(1); + out.hflush(); + checkSyncMetric(cluster, 0, 0); + checkSyncMetric(cluster, 1, 0); + checkSyncMetric(cluster, 2, 0); + out.hsync(); + checkSyncMetric(cluster, 0, 1); + checkSyncMetric(cluster, 1, 1); + checkSyncMetric(cluster, 2, 1); + out.hsync(); + checkSyncMetric(cluster, 0, 2); + checkSyncMetric(cluster, 1, 2); + checkSyncMetric(cluster, 2, 2); + } finally { + cluster.shutdown(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java index c2ecaf1d246869..e78ec230eb0f86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java @@ -61,104 +61,107 @@ private void testTriggerBlockReport(boolean incremental, boolean withSpecificNN) final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(1).build(); - cluster.waitActive(); - cluster.transitionToActive(0); - FileSystem fs = cluster.getFileSystem(0); - DatanodeProtocolClientSideTranslatorPB spyOnNn0 = - InternalDataNodeTestUtils.spyOnBposToNN( - cluster.getDataNodes().get(0), cluster.getNameNode(0)); - DatanodeProtocolClientSideTranslatorPB spyOnNn1 = - InternalDataNodeTestUtils.spyOnBposToNN( - cluster.getDataNodes().get(0), cluster.getNameNode(1)); - DFSTestUtil.createFile(fs, new Path("/abc"), 16, (short) 1, 1L); + try { + cluster.waitActive(); + cluster.transitionToActive(0); + FileSystem fs = cluster.getFileSystem(0); + DatanodeProtocolClientSideTranslatorPB spyOnNn0 = + InternalDataNodeTestUtils.spyOnBposToNN( + cluster.getDataNodes().get(0), cluster.getNameNode(0)); + DatanodeProtocolClientSideTranslatorPB spyOnNn1 = + InternalDataNodeTestUtils.spyOnBposToNN( + cluster.getDataNodes().get(0), cluster.getNameNode(1)); + DFSTestUtil.createFile(fs, new Path("/abc"), 16, (short) 1, 1L); - // We should get 1 incremental block report on both NNs. - Mockito.verify(spyOnNn0, timeout(60000).times(1)).blockReceivedAndDeleted( - any(DatanodeRegistration.class), - anyString(), - any(StorageReceivedDeletedBlocks[].class)); - Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReceivedAndDeleted( - any(DatanodeRegistration.class), - anyString(), - any(StorageReceivedDeletedBlocks[].class)); - - // We should not receive any more incremental or incremental block reports, - // since the interval we configured is so long. - for (int i = 0; i < 3; i++) { - Thread.sleep(10); - Mockito.verify(spyOnNn0, times(0)).blockReport( - any(DatanodeRegistration.class), - anyString(), - any(StorageBlockReport[].class), - any()); - Mockito.verify(spyOnNn0, times(1)).blockReceivedAndDeleted( + // We should get 1 incremental block report on both NNs. + Mockito.verify(spyOnNn0, timeout(60000).times(1)).blockReceivedAndDeleted( any(DatanodeRegistration.class), anyString(), any(StorageReceivedDeletedBlocks[].class)); - Mockito.verify(spyOnNn1, times(0)).blockReport( - any(DatanodeRegistration.class), - anyString(), - any(StorageBlockReport[].class), - any()); - Mockito.verify(spyOnNn1, times(1)).blockReceivedAndDeleted( + Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReceivedAndDeleted( any(DatanodeRegistration.class), anyString(), any(StorageReceivedDeletedBlocks[].class)); - } - // Create a fake block deletion notification on the DataNode. - // This will be sent with the next incremental block report. - ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo( - new Block(5678, 512, 1000), BlockStatus.DELETED_BLOCK, null); - DataNode datanode = cluster.getDataNodes().get(0); - for (BPServiceActor actor : datanode.getAllBpOs().get(0).getBPServiceActors()) { - final FsDatasetSpi dataset = datanode.getFSDataset(); - final DatanodeStorage storage; - try (FsDatasetSpi.FsVolumeReferences volumes = dataset.getFsVolumeReferences()) { - storage = dataset.getStorage(volumes.get(0).getStorageID()); + // We should not receive any more incremental or incremental block reports, + // since the interval we configured is so long. + for (int i = 0; i < 3; i++) { + Thread.sleep(10); + Mockito.verify(spyOnNn0, times(0)).blockReport( + any(DatanodeRegistration.class), + anyString(), + any(StorageBlockReport[].class), + any()); + Mockito.verify(spyOnNn0, times(1)).blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); + Mockito.verify(spyOnNn1, times(0)).blockReport( + any(DatanodeRegistration.class), + anyString(), + any(StorageBlockReport[].class), + any()); + Mockito.verify(spyOnNn1, times(1)).blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); } - actor.getIbrManager().addRDBI(rdbi, storage); - } - // Manually trigger a block report. - // Only trigger block report to NN1 when testing triggering block report on specific namenode. - InetSocketAddress nnAddr = withSpecificNN ? cluster.getNameNode(1).getServiceRpcAddress() : null; - datanode.triggerBlockReport( - new BlockReportOptions.Factory(). - setNamenodeAddr(nnAddr). - setIncremental(incremental). - build() - ); + // Create a fake block deletion notification on the DataNode. + // This will be sent with the next incremental block report. + ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo( + new Block(5678, 512, 1000), BlockStatus.DELETED_BLOCK, null); + DataNode datanode = cluster.getDataNodes().get(0); + for (BPServiceActor actor : datanode.getAllBpOs().get(0).getBPServiceActors()) { + final FsDatasetSpi dataset = datanode.getFSDataset(); + final DatanodeStorage storage; + try (FsDatasetSpi.FsVolumeReferences volumes = dataset.getFsVolumeReferences()) { + storage = dataset.getStorage(volumes.get(0).getStorageID()); + } + actor.getIbrManager().addRDBI(rdbi, storage); + } - // triggerBlockReport returns before the block report is - // actually sent. Wait for it to be sent here. - if (incremental) { - Mockito.verify(spyOnNn1, timeout(60000).times(2)). - blockReceivedAndDeleted( - any(DatanodeRegistration.class), - anyString(), - any(StorageReceivedDeletedBlocks[].class)); - int nn0IncrBlockReport = withSpecificNN ? 1 : 2; - Mockito.verify(spyOnNn0, timeout(60000).times(nn0IncrBlockReport)). - blockReceivedAndDeleted( - any(DatanodeRegistration.class), - anyString(), - any(StorageReceivedDeletedBlocks[].class)); - } else { - Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReport( - any(DatanodeRegistration.class), - anyString(), - any(StorageBlockReport[].class), - any()); - int nn0BlockReport = withSpecificNN ? 0 : 1; - Mockito.verify(spyOnNn0, timeout(60000).times(nn0BlockReport)).blockReport( - any(DatanodeRegistration.class), - anyString(), - any(StorageBlockReport[].class), - any()); - } + // Manually trigger a block report. + // Only trigger block report to NN1 when testing triggering block report on specific namenode. + InetSocketAddress nnAddr = + withSpecificNN ? cluster.getNameNode(1).getServiceRpcAddress() : null; + datanode.triggerBlockReport( + new BlockReportOptions.Factory(). + setNamenodeAddr(nnAddr). + setIncremental(incremental). + build() + ); - cluster.shutdown(); + // triggerBlockReport returns before the block report is + // actually sent. Wait for it to be sent here. + if (incremental) { + Mockito.verify(spyOnNn1, timeout(60000).times(2)). + blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); + int nn0IncrBlockReport = withSpecificNN ? 1 : 2; + Mockito.verify(spyOnNn0, timeout(60000).times(nn0IncrBlockReport)). + blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); + } else { + Mockito.verify(spyOnNn1, timeout(60000).times(1)).blockReport( + any(DatanodeRegistration.class), + anyString(), + any(StorageBlockReport[].class), + any()); + int nn0BlockReport = withSpecificNN ? 0 : 1; + Mockito.verify(spyOnNn0, timeout(60000).times(nn0BlockReport)).blockReport( + any(DatanodeRegistration.class), + anyString(), + any(StorageBlockReport[].class), + any()); + } + } finally { + cluster.shutdown(); + } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java index 0c6cb6751782d0..27e5c9a2083ee8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestProvidedImpl.java @@ -605,42 +605,43 @@ public void testScannerWithProvidedVolumes() throws Exception { public void testProvidedReplicaWithPathHandle() throws Exception { Configuration conf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.waitActive(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) { + cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); + DistributedFileSystem fs = cluster.getFileSystem(); - // generate random data - int chunkSize = 512; - Random r = new Random(12345L); - byte[] data = new byte[chunkSize]; - r.nextBytes(data); + // generate random data + int chunkSize = 512; + Random r = new Random(12345L); + byte[] data = new byte[chunkSize]; + r.nextBytes(data); - Path file = new Path("/testfile"); - try (FSDataOutputStream fout = fs.create(file)) { - fout.write(data); - } + Path file = new Path("/testfile"); + try (FSDataOutputStream fout = fs.create(file)) { + fout.write(data); + } - PathHandle pathHandle = fs.getPathHandle(fs.getFileStatus(file), - Options.HandleOpt.changed(true), Options.HandleOpt.moved(true)); - FinalizedProvidedReplica replica = new FinalizedProvidedReplica(0, - file.toUri(), 0, chunkSize, 0, pathHandle, null, conf, fs); - byte[] content = new byte[chunkSize]; - IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize); - assertArrayEquals(data, content); - - fs.rename(file, new Path("/testfile.1")); - // read should continue succeeding after the rename operation - IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize); - assertArrayEquals(data, content); - - replica.setPathHandle(null); - try { - // expected to fail as URI of the provided replica is no longer valid. - replica.getDataInputStream(0); - fail("Expected an exception"); - } catch (IOException e) { - LOG.info("Expected exception " + e); + PathHandle pathHandle = fs.getPathHandle(fs.getFileStatus(file), + Options.HandleOpt.changed(true), Options.HandleOpt.moved(true)); + FinalizedProvidedReplica replica = new FinalizedProvidedReplica(0, + file.toUri(), 0, chunkSize, 0, pathHandle, null, conf, fs); + byte[] content = new byte[chunkSize]; + IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize); + assertArrayEquals(data, content); + + fs.rename(file, new Path("/testfile.1")); + // read should continue succeeding after the rename operation + IOUtils.readFully(replica.getDataInputStream(0), content, 0, chunkSize); + assertArrayEquals(data, content); + + replica.setPathHandle(null); + try { + // expected to fail as URI of the provided replica is no longer valid. + replica.getDataInputStream(0); + fail("Expected an exception"); + } catch (IOException e) { + LOG.info("Expected exception " + e); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java index cf78c187ae2917..1765d8cc5235ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java @@ -572,40 +572,41 @@ public void testReplicaMapAfterDatanodeRestart() throws Exception { @Test public void testRecoverInconsistentRbw() throws IOException { Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, - new File(GenericTestUtils.getRandomizedTempPath())).build(); - cluster.waitActive(); - DataNode dn = cluster.getDataNodes().get(0); - FsDatasetImpl fsDataset = (FsDatasetImpl)DataNodeTestUtils.getFSDataset(dn); - - // set up replicasMap - String bpid = cluster.getNamesystem().getBlockPoolId(); - ExtendedBlock[] blocks = setup(bpid, cluster.getFsDatasetTestUtils(dn)); - - ReplicaBeingWritten rbw = (ReplicaBeingWritten)fsDataset. - getReplicaInfo(bpid, blocks[RBW].getBlockId()); - long bytesOnDisk = rbw.getBytesOnDisk(); - // simulate an inconsistent replica length update by reducing in-memory - // value of on disk length - rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null); - fsDataset.recoverRbw(blocks[RBW], blocks[RBW].getGenerationStamp(), 0L, - rbw.getNumBytes()); - // after the recovery, on disk length should equal acknowledged length. - assertTrue(rbw.getBytesOnDisk() == rbw.getBytesAcked()); - - // reduce on disk length again; this time actually truncate the file to - // simulate the data not being present - rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null); - try (RandomAccessFile blockRAF = rbw.getFileIoProvider(). - getRandomAccessFile(rbw.getVolume(), rbw.getBlockFile(), "rw")) { - // truncate blockFile - blockRAF.setLength(bytesOnDisk - 1); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf, + new File(GenericTestUtils.getRandomizedTempPath())).build()) { + cluster.waitActive(); + DataNode dn = cluster.getDataNodes().get(0); + FsDatasetImpl fsDataset = (FsDatasetImpl)DataNodeTestUtils.getFSDataset(dn); + + // set up replicasMap + String bpid = cluster.getNamesystem().getBlockPoolId(); + ExtendedBlock[] blocks = setup(bpid, cluster.getFsDatasetTestUtils(dn)); + + ReplicaBeingWritten rbw = (ReplicaBeingWritten)fsDataset. + getReplicaInfo(bpid, blocks[RBW].getBlockId()); + long bytesOnDisk = rbw.getBytesOnDisk(); + // simulate an inconsistent replica length update by reducing in-memory + // value of on disk length + rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null); fsDataset.recoverRbw(blocks[RBW], blocks[RBW].getGenerationStamp(), 0L, rbw.getNumBytes()); - fail("recovery should have failed"); - } catch (ReplicaNotFoundException rnfe) { - GenericTestUtils.assertExceptionContains("Found fewer bytesOnDisk than " + - "bytesAcked for replica", rnfe); + // after the recovery, on disk length should equal acknowledged length. + assertTrue(rbw.getBytesOnDisk() == rbw.getBytesAcked()); + + // reduce on disk length again; this time actually truncate the file to + // simulate the data not being present + rbw.setLastChecksumAndDataLen(bytesOnDisk - 1, null); + try (RandomAccessFile blockRAF = rbw.getFileIoProvider(). + getRandomAccessFile(rbw.getVolume(), rbw.getBlockFile(), "rw")) { + // truncate blockFile + blockRAF.setLength(bytesOnDisk - 1); + fsDataset.recoverRbw(blocks[RBW], blocks[RBW].getGenerationStamp(), 0L, + rbw.getNumBytes()); + fail("recovery should have failed"); + } catch (ReplicaNotFoundException rnfe) { + GenericTestUtils.assertExceptionContains("Found fewer bytesOnDisk than " + + "bytesAcked for replica", rnfe); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index 8e4e3dc9ae67b1..d5aae7fd8e7d71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -1451,47 +1451,48 @@ public void testMoverMetrics() throws Exception { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, blockSize); - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(2) .storageTypes( new StorageType[][] {{StorageType.DISK, StorageType.DISK}, {StorageType.ARCHIVE, StorageType.ARCHIVE}}) - .build(); + .build()) { + + cluster.waitActive(); + final DistributedFileSystem fs = cluster.getFileSystem(); + + final String file = "/testMaxIterationTime.dat"; + final Path path = new Path(file); + short repFactor = 1; + int seed = 0xFAFAFA; + // write to DISK + DFSTestUtil.createFile(fs, path, 4L * blockSize, repFactor, seed); - cluster.waitActive(); - final DistributedFileSystem fs = cluster.getFileSystem(); - - final String file = "/testMaxIterationTime.dat"; - final Path path = new Path(file); - short repFactor = 1; - int seed = 0xFAFAFA; - // write to DISK - DFSTestUtil.createFile(fs, path, 4L * blockSize, repFactor, seed); - - // move to ARCHIVE - fs.setStoragePolicy(new Path(file), "COLD"); - - Map> nnWithPath = new HashMap<>(); - List paths = new ArrayList<>(); - paths.add(path); - nnWithPath - .put(DFSUtil.getInternalNsRpcUris(conf).iterator().next(), paths); - - Mover.run(nnWithPath, conf); - - final String moverMetricsName = "Mover-" - + cluster.getNameNode(0).getNamesystem().getBlockPoolId(); - MetricsSource moverMetrics = - DefaultMetricsSystem.instance().getSource(moverMetricsName); - assertNotNull(moverMetrics); - - MetricsRecordBuilder rb = MetricsAsserts.getMetrics(moverMetricsName); - // Check metrics - assertEquals(4, MetricsAsserts.getLongCounter("BlocksScheduled", rb)); - assertEquals(1, MetricsAsserts.getLongCounter("FilesProcessed", rb)); - assertEquals(41943040, MetricsAsserts.getLongGauge("BytesMoved", rb)); - assertEquals(4, MetricsAsserts.getLongGauge("BlocksMoved", rb)); - assertEquals(0, MetricsAsserts.getLongGauge("BlocksFailed", rb)); + // move to ARCHIVE + fs.setStoragePolicy(new Path(file), "COLD"); + + Map> nnWithPath = new HashMap<>(); + List paths = new ArrayList<>(); + paths.add(path); + nnWithPath + .put(DFSUtil.getInternalNsRpcUris(conf).iterator().next(), paths); + + Mover.run(nnWithPath, conf); + + final String moverMetricsName = "Mover-" + + cluster.getNameNode(0).getNamesystem().getBlockPoolId(); + MetricsSource moverMetrics = + DefaultMetricsSystem.instance().getSource(moverMetricsName); + assertNotNull(moverMetrics); + + MetricsRecordBuilder rb = MetricsAsserts.getMetrics(moverMetricsName); + // Check metrics + assertEquals(4, MetricsAsserts.getLongCounter("BlocksScheduled", rb)); + assertEquals(1, MetricsAsserts.getLongCounter("FilesProcessed", rb)); + assertEquals(41943040, MetricsAsserts.getLongGauge("BytesMoved", rb)); + assertEquals(4, MetricsAsserts.getLongGauge("BlocksMoved", rb)); + assertEquals(0, MetricsAsserts.getLongGauge("BlocksFailed", rb)); + } } private void createFileWithFavoredDatanodes(final Configuration conf, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index ea4ed0fc9311e0..8da52f0629be70 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -1663,60 +1663,63 @@ DistributedFileSystem getDFS(MiniDFSCluster cluster, int nnIdx) public void testExpiryTimeConsistency() throws Exception { conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); - MiniDFSCluster dfsCluster = + try (MiniDFSCluster dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES) .nnTopology(MiniDFSNNTopology.simpleHATopology()) - .build(); - dfsCluster.transitionToActive(0); - - DistributedFileSystem fs = getDFS(dfsCluster, 0); - final NameNode ann = dfsCluster.getNameNode(0); - - final Path filename = new Path("/file"); - final short replication = (short) 3; - DFSTestUtil.createFile(fs, filename, 1, replication, 0x0BAC); - fs.addCachePool(new CachePoolInfo("pool")); - long id = fs.addCacheDirective( - new CacheDirectiveInfo.Builder().setPool("pool").setPath(filename) - .setExpiration(CacheDirectiveInfo.Expiration.newRelative(86400000)) - .setReplication(replication).build()); - fs.modifyCacheDirective(new CacheDirectiveInfo.Builder() - .setId(id) - .setExpiration(CacheDirectiveInfo.Expiration.newRelative(172800000)) - .build()); - final NameNode sbn = dfsCluster.getNameNode(1); - final CacheManager annCachemanager = ann.getNamesystem().getCacheManager(); - final CacheManager sbnCachemanager = sbn.getNamesystem().getCacheManager(); - HATestUtil.waitForStandbyToCatchUp(ann, sbn); - GenericTestUtils.waitFor(() -> { - boolean isConsistence = false; - ann.getNamesystem().readLock(RwLockMode.FS); - try { - sbn.getNamesystem().readLock(RwLockMode.FS); + .build()) { + dfsCluster.transitionToActive(0); + + DistributedFileSystem fs = getDFS(dfsCluster, 0); + final NameNode ann = dfsCluster.getNameNode(0); + + final Path filename = new Path("/file"); + final short replication = (short) 3; + DFSTestUtil.createFile(fs, filename, 1, replication, 0x0BAC); + fs.addCachePool(new CachePoolInfo("pool")); + long id = fs.addCacheDirective( + new CacheDirectiveInfo.Builder().setPool("pool").setPath(filename) + .setExpiration(CacheDirectiveInfo.Expiration.newRelative(86400000)) + .setReplication(replication).build()); + fs.modifyCacheDirective(new CacheDirectiveInfo.Builder() + .setId(id) + .setExpiration(CacheDirectiveInfo.Expiration.newRelative(172800000)) + .build()); + final NameNode sbn = dfsCluster.getNameNode(1); + final CacheManager annCachemanager = ann.getNamesystem().getCacheManager(); + final CacheManager sbnCachemanager = sbn.getNamesystem().getCacheManager(); + HATestUtil.waitForStandbyToCatchUp(ann, sbn); + GenericTestUtils.waitFor(() -> { + boolean isConsistence = false; + ann.getNamesystem().readLock(RwLockMode.FS); try { - Iterator annDirectivesIt = annCachemanager. - getCacheDirectives().iterator(); - Iterator sbnDirectivesIt = sbnCachemanager. - getCacheDirectives().iterator(); - if (annDirectivesIt.hasNext() && sbnDirectivesIt.hasNext()) { - CacheDirective annDirective = annDirectivesIt.next(); - CacheDirective sbnDirective = sbnDirectivesIt.next(); - if (annDirective.getExpiryTimeString(). - equals(sbnDirective.getExpiryTimeString())) { - isConsistence = true; + sbn.getNamesystem().readLock(RwLockMode.FS); + try { + Iterator annDirectivesIt = annCachemanager. + getCacheDirectives().iterator(); + Iterator sbnDirectivesIt = sbnCachemanager. + getCacheDirectives().iterator(); + if (annDirectivesIt.hasNext() && sbnDirectivesIt.hasNext()) { + CacheDirective annDirective = annDirectivesIt.next(); + CacheDirective sbnDirective = sbnDirectivesIt.next(); + if (annDirective.getExpiryTimeString(). + equals(sbnDirective.getExpiryTimeString())) { + isConsistence = true; + } } + } finally { + sbn.getNamesystem().readUnlock(RwLockMode.FS, + "expiryTimeConsistency"); } } finally { - sbn.getNamesystem().readUnlock(RwLockMode.FS, "expiryTimeConsistency"); + ann.getNamesystem().readUnlock(RwLockMode.FS, + "expiryTimeConsistency"); } - } finally { - ann.getNamesystem().readUnlock(RwLockMode.FS, "expiryTimeConsistency"); - } - if (!isConsistence) { - LOG.info("testEexpiryTimeConsistency:" - + "ANN CacheDirective Status is inconsistent with SBN"); - } - return isConsistence; - }, 500, 120000); + if (!isConsistence) { + LOG.info("testEexpiryTimeConsistency:" + + "ANN CacheDirective Status is inconsistent with SBN"); + } + return isConsistence; + }, 500, 120000); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java index 64df2e6f1147f1..ec6bb6cdfc0312 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCorrectnessOfQuotaAfterRenameOp.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.PathUtils; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -52,6 +53,14 @@ public static void setUp() throws IOException { dfs = cluster.getFileSystem(); } + @AfterAll + public static void tearDown() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + @Test public void testQuotaUsageWhenRenameWithSameStoragePolicy() throws Exception { final int fileLen = 1024; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 8967001b7b9bea..31510a3efc7c06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -1168,59 +1168,60 @@ private void ensureSubSectionsAlignWithParent(ArrayList
subSec, @Test public void testUpdateBlocksMapAndNameCacheAsync() throws IOException { Configuration conf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); - cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); - FSDirectory fsdir = cluster.getNameNode().namesystem.getFSDirectory(); - File workingDir = GenericTestUtils.getTestDir(); - - File preRestartTree = new File(workingDir, "preRestartTree"); - File postRestartTree = new File(workingDir, "postRestartTree"); - - Path baseDir = new Path("/user/foo"); - fs.mkdirs(baseDir); - fs.allowSnapshot(baseDir); - for (int i = 0; i < 5; i++) { - Path dir = new Path(baseDir, Integer.toString(i)); - fs.mkdirs(dir); - for (int j = 0; j < 5; j++) { - Path file = new Path(dir, Integer.toString(j)); - FSDataOutputStream os = fs.create(file); - os.write((byte) j); - os.close(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()) { + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + FSDirectory fsdir = cluster.getNameNode().namesystem.getFSDirectory(); + File workingDir = GenericTestUtils.getTestDir(); + + File preRestartTree = new File(workingDir, "preRestartTree"); + File postRestartTree = new File(workingDir, "postRestartTree"); + + Path baseDir = new Path("/user/foo"); + fs.mkdirs(baseDir); + fs.allowSnapshot(baseDir); + for (int i = 0; i < 5; i++) { + Path dir = new Path(baseDir, Integer.toString(i)); + fs.mkdirs(dir); + for (int j = 0; j < 5; j++) { + Path file = new Path(dir, Integer.toString(j)); + FSDataOutputStream os = fs.create(file); + os.write((byte) j); + os.close(); + } + fs.createSnapshot(baseDir, "snap_"+i); + fs.rename(new Path(dir, "0"), new Path(dir, "renamed")); } - fs.createSnapshot(baseDir, "snap_"+i); - fs.rename(new Path(dir, "0"), new Path(dir, "renamed")); - } - SnapshotTestHelper.dumpTree2File(fsdir, preRestartTree); - - // checkpoint - fs.setSafeMode(SafeModeAction.ENTER); - fs.saveNamespace(); - fs.setSafeMode(SafeModeAction.LEAVE); + SnapshotTestHelper.dumpTree2File(fsdir, preRestartTree); - cluster.restartNameNode(); - cluster.waitActive(); - fs = cluster.getFileSystem(); - fsdir = cluster.getNameNode().namesystem.getFSDirectory(); + // checkpoint + fs.setSafeMode(SafeModeAction.ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.LEAVE); - // Ensure all the files created above exist, and blocks is correct. - for (int i = 0; i < 5; i++) { - Path dir = new Path(baseDir, Integer.toString(i)); - assertTrue(fs.getFileStatus(dir).isDirectory()); - for (int j = 0; j < 5; j++) { - Path file = new Path(dir, Integer.toString(j)); - if (j == 0) { - file = new Path(dir, "renamed"); + cluster.restartNameNode(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fsdir = cluster.getNameNode().namesystem.getFSDirectory(); + + // Ensure all the files created above exist, and blocks is correct. + for (int i = 0; i < 5; i++) { + Path dir = new Path(baseDir, Integer.toString(i)); + assertTrue(fs.getFileStatus(dir).isDirectory()); + for (int j = 0; j < 5; j++) { + Path file = new Path(dir, Integer.toString(j)); + if (j == 0) { + file = new Path(dir, "renamed"); + } + FSDataInputStream in = fs.open(file); + int n = in.readByte(); + assertEquals(j, n); + in.close(); } - FSDataInputStream in = fs.open(file); - int n = in.readByte(); - assertEquals(j, n); - in.close(); } + SnapshotTestHelper.dumpTree2File(fsdir, postRestartTree); + SnapshotTestHelper.compareDumpedTreeInFile( + preRestartTree, postRestartTree, true); } - SnapshotTestHelper.dumpTree2File(fsdir, postRestartTree); - SnapshotTestHelper.compareDumpedTreeInFile( - preRestartTree, postRestartTree, true); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java index 87a582c02598df..6bee0b275f8fe0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java @@ -52,38 +52,39 @@ public void testPrintTopologyTextFormat() throws IOException { } } - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(dataNodesNum) .racks(rackList.toArray(new String[rackList.size()])) - .build(); - cluster.waitActive(); - - // get http uri - String httpUri = cluster.getHttpUri(0); - - // send http request - URL url = new URL(httpUri + "/topology"); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setReadTimeout(20000); - conn.setConnectTimeout(20000); - conn.connect(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); - StringBuilder sb = - new StringBuilder("-- Network Topology -- \n"); - sb.append(out); - sb.append("\n-- Network Topology -- "); - String topology = sb.toString(); - - // assert rack info - assertTrue(topology.contains("/rack0")); - assertTrue(topology.contains("/rack1")); - assertTrue(topology.contains("/rack2")); - assertTrue(topology.contains("/rack3")); - assertTrue(topology.contains("/rack4")); - - // assert node number - assertEquals(topology.split("127.0.0.1").length - 1, dataNodesNum); + .build()) { + cluster.waitActive(); + + // get http uri + String httpUri = cluster.getHttpUri(0); + + // send http request + URL url = new URL(httpUri + "/topology"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(20000); + conn.setConnectTimeout(20000); + conn.connect(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); + StringBuilder sb = + new StringBuilder("-- Network Topology -- \n"); + sb.append(out); + sb.append("\n-- Network Topology -- "); + String topology = sb.toString(); + + // assert rack info + assertTrue(topology.contains("/rack0")); + assertTrue(topology.contains("/rack1")); + assertTrue(topology.contains("/rack2")); + assertTrue(topology.contains("/rack3")); + assertTrue(topology.contains("/rack4")); + + // assert node number + assertEquals(topology.split("127.0.0.1").length - 1, dataNodesNum); + } } @Test @@ -99,10 +100,53 @@ public void testPrintTopologyJsonFormat() throws IOException { } } - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(dataNodesNum) .racks(rackList.toArray(new String[rackList.size()])) - .build(); + .build()) { + cluster.waitActive(); + + // get http uri + String httpUri = cluster.getHttpUri(0); + + // send http request + URL url = new URL(httpUri + "/topology"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(20000); + conn.setConnectTimeout(20000); + conn.setRequestProperty("Accept", "application/json"); + conn.connect(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); + String topology = out.toString(); + + // parse json + JsonNode racks = new ObjectMapper().readTree(topology); + + // assert rack number + assertEquals(racks.size(), 5); + + // assert node number + Iterator elements = racks.elements(); + int dataNodesCount = 0; + while(elements.hasNext()){ + JsonNode rack = elements.next(); + Iterator> fields = rack.fields(); + while (fields.hasNext()) { + dataNodesCount += fields.next().getValue().size(); + } + } + assertEquals(dataNodesCount, dataNodesNum); + } + } + + @Test + public void testPrintTopologyNoDatanodesTextFormat() throws IOException { + StaticMapping.resetMap(); + Configuration conf = new HdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) + .build()) { cluster.waitActive(); // get http uri @@ -113,89 +157,49 @@ public void testPrintTopologyJsonFormat() throws IOException { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setReadTimeout(20000); conn.setConnectTimeout(20000); - conn.setRequestProperty("Accept", "application/json"); conn.connect(); ByteArrayOutputStream out = new ByteArrayOutputStream(); IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); - String topology = out.toString(); - - // parse json - JsonNode racks = new ObjectMapper().readTree(topology); - - // assert rack number - assertEquals(racks.size(), 5); + StringBuilder sb = + new StringBuilder("-- Network Topology -- \n"); + sb.append(out); + sb.append("\n-- Network Topology -- "); + String topology = sb.toString(); // assert node number - Iterator elements = racks.elements(); - int dataNodesCount = 0; - while(elements.hasNext()){ - JsonNode rack = elements.next(); - Iterator> fields = rack.fields(); - while (fields.hasNext()) { - dataNodesCount += fields.next().getValue().size(); - } - } - assertEquals(dataNodesCount, dataNodesNum); - } - - @Test - public void testPrintTopologyNoDatanodesTextFormat() throws IOException { - StaticMapping.resetMap(); - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(0) - .build(); - cluster.waitActive(); - - // get http uri - String httpUri = cluster.getHttpUri(0); - - // send http request - URL url = new URL(httpUri + "/topology"); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setReadTimeout(20000); - conn.setConnectTimeout(20000); - conn.connect(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); - StringBuilder sb = - new StringBuilder("-- Network Topology -- \n"); - sb.append(out); - sb.append("\n-- Network Topology -- "); - String topology = sb.toString(); - - // assert node number - assertTrue(topology.contains("No DataNodes")); + assertTrue(topology.contains("No DataNodes")); + } } @Test public void testPrintTopologyNoDatanodesJsonFormat() throws IOException { StaticMapping.resetMap(); Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(0) - .build(); - cluster.waitActive(); - - // get http uri - String httpUri = cluster.getHttpUri(0); - - // send http request - URL url = new URL(httpUri + "/topology"); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setReadTimeout(20000); - conn.setConnectTimeout(20000); - conn.setRequestProperty("Accept", "application/json"); - conn.connect(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); - StringBuilder sb = - new StringBuilder("-- Network Topology -- \n"); - sb.append(out); - sb.append("\n-- Network Topology -- "); - String topology = sb.toString(); - - // assert node number - assertTrue(topology.contains("No DataNodes")); + .build()) { + cluster.waitActive(); + + // get http uri + String httpUri = cluster.getHttpUri(0); + + // send http request + URL url = new URL(httpUri + "/topology"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(20000); + conn.setConnectTimeout(20000); + conn.setRequestProperty("Accept", "application/json"); + conn.connect(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); + StringBuilder sb = + new StringBuilder("-- Network Topology -- \n"); + sb.append(out); + sb.append("\n-- Network Topology -- "); + String topology = sb.toString(); + + // assert node number + assertTrue(topology.contains("No DataNodes")); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java index c76c759cc24e3a..9fe2696c227ecb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestProtectedDirectories.java @@ -222,34 +222,38 @@ public void testReconfigureProtectedPaths() throws Throwable { MiniDFSCluster cluster = setupTestCase(conf, protectedPaths, unprotectedPaths); - SortedSet protectedPathsNew = new TreeSet<>( - FSDirectory.normalizePaths(Arrays.asList("/aa", "/bb", "/cc"), - FS_PROTECTED_DIRECTORIES)); + try { + SortedSet protectedPathsNew = new TreeSet<>( + FSDirectory.normalizePaths(Arrays.asList("/aa", "/bb", "/cc"), + FS_PROTECTED_DIRECTORIES)); - String protectedPathsStrNew = "/aa,/bb,/cc"; + String protectedPathsStrNew = "/aa,/bb,/cc"; - NameNode nn = cluster.getNameNode(); + NameNode nn = cluster.getNameNode(); - // change properties - nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, protectedPathsStrNew); + // change properties + nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, protectedPathsStrNew); - FSDirectory fsDirectory = nn.getNamesystem().getFSDirectory(); - // verify change - assertEquals(protectedPathsNew, fsDirectory.getProtectedDirectories(), - String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); + FSDirectory fsDirectory = nn.getNamesystem().getFSDirectory(); + // verify change + assertEquals(protectedPathsNew, fsDirectory.getProtectedDirectories(), + String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); - assertEquals(protectedPathsStrNew, nn.getConf().get(FS_PROTECTED_DIRECTORIES), - String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); + assertEquals(protectedPathsStrNew, nn.getConf().get(FS_PROTECTED_DIRECTORIES), + String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); - // revert to default - nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, null); + // revert to default + nn.reconfigureProperty(FS_PROTECTED_DIRECTORIES, null); - // verify default - assertEquals(new TreeSet(), fsDirectory.getProtectedDirectories(), - String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); + // verify default + assertEquals(new TreeSet(), fsDirectory.getProtectedDirectories(), + String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); - assertEquals(null, nn.getConf().get(FS_PROTECTED_DIRECTORIES), - String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); + assertEquals(null, nn.getConf().get(FS_PROTECTED_DIRECTORIES), + String.format("%s has wrong value", FS_PROTECTED_DIRECTORIES)); + } finally { + cluster.shutdown(); + } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java index 76a83f6a27595c..45f7675a8d254d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapAliasmap.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.server.namenode.TransferFsImage; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -60,6 +61,14 @@ public void setup() throws Exception { cluster.waitActive(); } + @AfterEach + public void tearDown() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } + @Test public void testAliasmapBootstrap() throws Exception { InMemoryLevelDBAliasMapServer aliasMapServer = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java index 39b6bf8aa32f3f..405c8ccf25e536 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAMetrics.java @@ -192,36 +192,38 @@ public void testGetNameNodeState() throws IOException { conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, Integer.MAX_VALUE); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( - MiniDFSNNTopology.simpleHATopology(3)).numDataNodes(1).build(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology( + MiniDFSNNTopology.simpleHATopology(3)).numDataNodes(1).build()) { - cluster.waitActive(); + cluster.waitActive(); - NameNode nn0 = cluster.getNameNode(0); - NameNode nn1 = cluster.getNameNode(1); - NameNode nn2 = cluster.getNameNode(2); + NameNode nn0 = cluster.getNameNode(0); + NameNode nn1 = cluster.getNameNode(1); + NameNode nn2 = cluster.getNameNode(2); - // All namenodes are in standby by default - assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState()); - assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn1.getNameNodeState()); - assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn2.getNameNodeState()); + // All namenodes are in standby by default + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState()); + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn1.getNameNodeState()); + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn2.getNameNodeState()); - // Transition nn0 to be active - cluster.transitionToActive(0); - assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn0.getNameNodeState()); + // Transition nn0 to be active + cluster.transitionToActive(0); + assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn0.getNameNodeState()); - // Transition nn1 to be active - cluster.transitionToStandby(0); - cluster.transitionToActive(1); - assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState()); - assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn1.getNameNodeState()); + // Transition nn1 to be active + cluster.transitionToStandby(0); + cluster.transitionToActive(1); + assertEquals(HAServiceProtocol.HAServiceState.STANDBY.ordinal(), nn0.getNameNodeState()); + assertEquals(HAServiceProtocol.HAServiceState.ACTIVE.ordinal(), nn1.getNameNodeState()); - // Transition nn2 to observer - cluster.transitionToObserver(2); - assertEquals(HAServiceProtocol.HAServiceState.OBSERVER.ordinal(), nn2.getNameNodeState()); + // Transition nn2 to observer + cluster.transitionToObserver(2); + assertEquals(HAServiceProtocol.HAServiceState.OBSERVER.ordinal(), nn2.getNameNodeState()); - // Shutdown nn2. Now getNameNodeState should return the INITIALIZING state. - cluster.shutdownNameNode(2); - assertEquals(HAServiceProtocol.HAServiceState.INITIALIZING.ordinal(), nn2.getNameNodeState()); + // Shutdown nn2. Now getNameNodeState should return the INITIALIZING state. + cluster.shutdownNameNode(2); + assertEquals(HAServiceProtocol.HAServiceState.INITIALIZING.ordinal(), + nn2.getNameNodeState()); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index b271069768832e..eaa1a6069b6e93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -818,6 +818,7 @@ public void testTransactionSinceLastCheckpointMetrics() throws Exception { Random random = new Random(); int retryCount = 0; while (retryCount < 5) { + MiniDFSCluster cluster2 = null; try { int basePort = 10060 + random.nextInt(100) * 2; MiniDFSNNTopology topology = new MiniDFSNNTopology() @@ -836,7 +837,7 @@ public void testTransactionSinceLastCheckpointMetrics() throws Exception { 1); // Poll and follow ANN txns very often, for purpose of testing. conf2.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); - MiniDFSCluster cluster2 = new MiniDFSCluster.Builder(conf2) + cluster2 = new MiniDFSCluster.Builder(conf2) .nnTopology(topology).numDataNodes(1).build(); cluster2.waitActive(); DistributedFileSystem fs2 = cluster2.getFileSystem(0); @@ -877,11 +878,14 @@ public void testTransactionSinceLastCheckpointMetrics() throws Exception { cluster2.getNameNode(1).getNamesystem() .getTransactionsSinceLastCheckpoint(), "SBN failed to track 2 added txns after the ckpt."); - cluster2.shutdown(); break; } catch (Exception e) { LOG.warn("Unable to set up HA cluster, exception thrown: " + e); retryCount++; + } finally { + if (cluster2 != null) { + cluster2.shutdown(); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index bf46133868279e..de0c05d397f7f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -459,67 +459,70 @@ public void testAllocShm() throws Exception { Configuration conf = createShortCircuitConf("testAllocShm", sockDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); - final ShortCircuitCache cache = - fs.getClient().getClientContext().getShortCircuitCache(0); - cache.getDfsClientShmManager().visit(new Visitor() { - @Override - public void visit(HashMap info) - throws IOException { - // The ClientShmManager starts off empty - assertEquals(0, info.size()); - } - }); - DomainPeer peer = getDomainPeerToDn(conf); - MutableBoolean usedPeer = new MutableBoolean(false); - ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz"); - final DatanodeInfo datanode = new DatanodeInfoBuilder() - .setNodeID(cluster.getDataNodes().get(0).getDatanodeId()) - .build(); - // Allocating the first shm slot requires using up a peer. - Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, - blockId, "testAllocShm_client"); - assertNotNull(slot); - assertTrue(usedPeer.booleanValue()); - cache.getDfsClientShmManager().visit(new Visitor() { - @Override - public void visit(HashMap info) - throws IOException { - // The ClientShmManager starts off empty - assertEquals(1, info.size()); - PerDatanodeVisitorInfo vinfo = info.get(datanode); - assertFalse(vinfo.disabled); - assertEquals(0, vinfo.full.size()); - assertEquals(1, vinfo.notFull.size()); - } - }); - cache.scheduleSlotReleaser(slot); - // Wait for the slot to be released, and the shared memory area to be - // closed. Since we didn't register this shared memory segment on the - // server, it will also be a test of how well the server deals with - // bogus client behavior. - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - final MutableBoolean done = new MutableBoolean(false); - try { - cache.getDfsClientShmManager().visit(new Visitor() { - @Override - public void visit(HashMap info) - throws IOException { - done.setValue(info.get(datanode).full.isEmpty() && - info.get(datanode).notFull.isEmpty()); - } - }); - } catch (IOException e) { - LOG.error("error running visitor", e); + try { + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + final ShortCircuitCache cache = + fs.getClient().getClientContext().getShortCircuitCache(0); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + // The ClientShmManager starts off empty + assertEquals(0, info.size()); } - return done.booleanValue(); - } - }, 10, 60000); - cluster.shutdown(); - sockDir.close(); + }); + DomainPeer peer = getDomainPeerToDn(conf); + MutableBoolean usedPeer = new MutableBoolean(false); + ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz"); + final DatanodeInfo datanode = new DatanodeInfoBuilder() + .setNodeID(cluster.getDataNodes().get(0).getDatanodeId()) + .build(); + // Allocating the first shm slot requires using up a peer. + Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, + blockId, "testAllocShm_client"); + assertNotNull(slot); + assertTrue(usedPeer.booleanValue()); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + // The ClientShmManager starts off empty + assertEquals(1, info.size()); + PerDatanodeVisitorInfo vinfo = info.get(datanode); + assertFalse(vinfo.disabled); + assertEquals(0, vinfo.full.size()); + assertEquals(1, vinfo.notFull.size()); + } + }); + cache.scheduleSlotReleaser(slot); + // Wait for the slot to be released, and the shared memory area to be + // closed. Since we didn't register this shared memory segment on the + // server, it will also be a test of how well the server deals with + // bogus client behavior. + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + final MutableBoolean done = new MutableBoolean(false); + try { + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + done.setValue(info.get(datanode).full.isEmpty() && + info.get(datanode).notFull.isEmpty()); + } + }); + } catch (IOException e) { + LOG.error("error running visitor", e); + } + return done.booleanValue(); + } + }, 10, 60000); + } finally { + cluster.shutdown(); + sockDir.close(); + } } @Test @@ -530,51 +533,54 @@ public void testShmBasedStaleness() throws Exception { Configuration conf = createShortCircuitConf("testShmBasedStaleness", sockDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); - final ShortCircuitCache cache = - fs.getClient().getClientContext().getShortCircuitCache(0); - String TEST_FILE = "/test_file"; - final int TEST_FILE_LEN = 8193; - final int SEED = 0xFADED; - DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, - (short)1, SEED); - FSDataInputStream fis = fs.open(new Path(TEST_FILE)); - int first = fis.read(); - final ExtendedBlock block = - DFSTestUtil.getFirstBlock(fs, new Path(TEST_FILE)); - assertTrue(first != -1); - cache.accept(new CacheVisitor() { - @Override - public void visit(int numOutstandingMmaps, - Map replicas, - Map failedLoads, - LinkedMap evictable, - LinkedMap evictableMmapped) { - ShortCircuitReplica replica = replicas.get( - ExtendedBlockId.fromExtendedBlock(block)); - assertNotNull(replica); - assertTrue(replica.getSlot().isValid()); - } - }); - // Stop the Namenode. This will close the socket keeping the client's - // shared memory segment alive, and make it stale. - cluster.getDataNodes().get(0).shutdown(); - cache.accept(new CacheVisitor() { - @Override - public void visit(int numOutstandingMmaps, - Map replicas, - Map failedLoads, - LinkedMap evictable, - LinkedMap evictableMmapped) { - ShortCircuitReplica replica = replicas.get( - ExtendedBlockId.fromExtendedBlock(block)); - assertNotNull(replica); - assertFalse(replica.getSlot().isValid()); - } - }); - cluster.shutdown(); - sockDir.close(); + try { + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + final ShortCircuitCache cache = + fs.getClient().getClientContext().getShortCircuitCache(0); + String TEST_FILE = "/test_file"; + final int TEST_FILE_LEN = 8193; + final int SEED = 0xFADED; + DFSTestUtil.createFile(fs, new Path(TEST_FILE), TEST_FILE_LEN, + (short)1, SEED); + FSDataInputStream fis = fs.open(new Path(TEST_FILE)); + int first = fis.read(); + final ExtendedBlock block = + DFSTestUtil.getFirstBlock(fs, new Path(TEST_FILE)); + assertTrue(first != -1); + cache.accept(new CacheVisitor() { + @Override + public void visit(int numOutstandingMmaps, + Map replicas, + Map failedLoads, + LinkedMap evictable, + LinkedMap evictableMmapped) { + ShortCircuitReplica replica = replicas.get( + ExtendedBlockId.fromExtendedBlock(block)); + assertNotNull(replica); + assertTrue(replica.getSlot().isValid()); + } + }); + // Stop the Namenode. This will close the socket keeping the client's + // shared memory segment alive, and make it stale. + cluster.getDataNodes().get(0).shutdown(); + cache.accept(new CacheVisitor() { + @Override + public void visit(int numOutstandingMmaps, + Map replicas, + Map failedLoads, + LinkedMap evictable, + LinkedMap evictableMmapped) { + ShortCircuitReplica replica = replicas.get( + ExtendedBlockId.fromExtendedBlock(block)); + assertNotNull(replica); + assertFalse(replica.getSlot().isValid()); + } + }); + } finally { + cluster.shutdown(); + sockDir.close(); + } } /** @@ -595,82 +601,85 @@ public void testUnlinkingReplicasInFileDescriptorCache() throws Exception { 1000000000L); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); - final ShortCircuitCache cache = - fs.getClient().getClientContext().getShortCircuitCache(0); - cache.getDfsClientShmManager().visit(new Visitor() { - @Override - public void visit(HashMap info) - throws IOException { - // The ClientShmManager starts off empty. - assertEquals(0, info.size()); - } - }); - final Path TEST_PATH = new Path("/test_file"); - final int TEST_FILE_LEN = 8193; - final int SEED = 0xFADE0; - DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN, - (short)1, SEED); - byte contents[] = DFSTestUtil.readFileBuffer(fs, TEST_PATH); - byte expected[] = DFSTestUtil. - calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); - assertTrue(Arrays.equals(contents, expected)); - // Loading this file brought the ShortCircuitReplica into our local - // replica cache. - final DatanodeInfo datanode = new DatanodeInfoBuilder() - .setNodeID(cluster.getDataNodes().get(0).getDatanodeId()) - .build(); - cache.getDfsClientShmManager().visit(new Visitor() { - @Override - public void visit(HashMap info) - throws IOException { - assertTrue(info.get(datanode).full.isEmpty()); - assertFalse(info.get(datanode).disabled); - assertEquals(1, info.get(datanode).notFull.values().size()); - DfsClientShm shm = - info.get(datanode).notFull.values().iterator().next(); - assertFalse(shm.isDisconnected()); - } - }); - // Remove the file whose blocks we just read. - fs.delete(TEST_PATH, false); + try { + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + final ShortCircuitCache cache = + fs.getClient().getClientContext().getShortCircuitCache(0); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + // The ClientShmManager starts off empty. + assertEquals(0, info.size()); + } + }); + final Path TEST_PATH = new Path("/test_file"); + final int TEST_FILE_LEN = 8193; + final int SEED = 0xFADE0; + DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN, + (short)1, SEED); + byte contents[] = DFSTestUtil.readFileBuffer(fs, TEST_PATH); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + assertTrue(Arrays.equals(contents, expected)); + // Loading this file brought the ShortCircuitReplica into our local + // replica cache. + final DatanodeInfo datanode = new DatanodeInfoBuilder() + .setNodeID(cluster.getDataNodes().get(0).getDatanodeId()) + .build(); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + assertTrue(info.get(datanode).full.isEmpty()); + assertFalse(info.get(datanode).disabled); + assertEquals(1, info.get(datanode).notFull.values().size()); + DfsClientShm shm = + info.get(datanode).notFull.values().iterator().next(); + assertFalse(shm.isDisconnected()); + } + }); + // Remove the file whose blocks we just read. + fs.delete(TEST_PATH, false); - // Wait for the replica to be purged from the DFSClient's cache. - GenericTestUtils.waitFor(new Supplier() { - MutableBoolean done = new MutableBoolean(true); - @Override - public Boolean get() { - try { - done.setValue(true); - cache.getDfsClientShmManager().visit(new Visitor() { - @Override - public void visit(HashMap info) throws IOException { - assertTrue(info.get(datanode).full.isEmpty()); - assertFalse(info.get(datanode).disabled); - assertEquals(1, - info.get(datanode).notFull.values().size()); - DfsClientShm shm = info.get(datanode).notFull.values(). - iterator().next(); - // Check that all slots have been invalidated. - for (Iterator iter = shm.slotIterator(); - iter.hasNext(); ) { - Slot slot = iter.next(); - if (slot.isValid()) { - done.setValue(false); + // Wait for the replica to be purged from the DFSClient's cache. + GenericTestUtils.waitFor(new Supplier() { + MutableBoolean done = new MutableBoolean(true); + @Override + public Boolean get() { + try { + done.setValue(true); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) throws IOException { + assertTrue(info.get(datanode).full.isEmpty()); + assertFalse(info.get(datanode).disabled); + assertEquals(1, + info.get(datanode).notFull.values().size()); + DfsClientShm shm = info.get(datanode).notFull.values(). + iterator().next(); + // Check that all slots have been invalidated. + for (Iterator iter = shm.slotIterator(); + iter.hasNext(); ) { + Slot slot = iter.next(); + if (slot.isValid()) { + done.setValue(false); + } } } - } - }); - } catch (IOException e) { - LOG.error("error running visitor", e); + }); + } catch (IOException e) { + LOG.error("error running visitor", e); + } + return done.booleanValue(); } - return done.booleanValue(); - } - }, 10, 60000); - cluster.shutdown(); - sockDir.close(); + }, 10, 60000); + } finally { + cluster.shutdown(); + sockDir.close(); + } } static private void checkNumberOfSegmentsAndSlots(final int expectedSegments, @@ -713,34 +722,37 @@ public void testDataXceiverCleansUpSlotsOnFailure() throws Exception { 1000000000L); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); - final Path TEST_PATH1 = new Path("/test_file1"); - final Path TEST_PATH2 = new Path("/test_file2"); - final int TEST_FILE_LEN = 4096; - final int SEED = 0xFADE1; - DFSTestUtil.createFile(fs, TEST_PATH1, TEST_FILE_LEN, - (short)1, SEED); - DFSTestUtil.createFile(fs, TEST_PATH2, TEST_FILE_LEN, - (short)1, SEED); - - // The first read should allocate one shared memory segment and slot. - DFSTestUtil.readFileBuffer(fs, TEST_PATH1); - - // The second read should fail, and we should only have 1 segment and 1 slot - // left. - BlockReaderFactory.setFailureInjectorForTesting( - new TestCleanupFailureInjector()); try { - DFSTestUtil.readFileBuffer(fs, TEST_PATH2); - } catch (Throwable t) { - GenericTestUtils.assertExceptionContains("TCP reads were disabled for " + - "testing, but we failed to do a non-TCP read.", t); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + final Path TEST_PATH1 = new Path("/test_file1"); + final Path TEST_PATH2 = new Path("/test_file2"); + final int TEST_FILE_LEN = 4096; + final int SEED = 0xFADE1; + DFSTestUtil.createFile(fs, TEST_PATH1, TEST_FILE_LEN, + (short)1, SEED); + DFSTestUtil.createFile(fs, TEST_PATH2, TEST_FILE_LEN, + (short)1, SEED); + + // The first read should allocate one shared memory segment and slot. + DFSTestUtil.readFileBuffer(fs, TEST_PATH1); + + // The second read should fail, and we should only have 1 segment and 1 slot + // left. + BlockReaderFactory.setFailureInjectorForTesting( + new TestCleanupFailureInjector()); + try { + DFSTestUtil.readFileBuffer(fs, TEST_PATH2); + } catch (Throwable t) { + GenericTestUtils.assertExceptionContains("TCP reads were disabled for " + + "testing, but we failed to do a non-TCP read.", t); + } + checkNumberOfSegmentsAndSlots(1, 1, + cluster.getDataNodes().get(0).getShortCircuitRegistry()); + } finally { + cluster.shutdown(); + sockDir.close(); } - checkNumberOfSegmentsAndSlots(1, 1, - cluster.getDataNodes().get(0).getShortCircuitRegistry()); - cluster.shutdown(); - sockDir.close(); } // Regression test for HADOOP-11802 @@ -756,51 +768,53 @@ public void testDataXceiverHandlesRequestShortCircuitShmFailure() 1000000000L); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); - final Path TEST_PATH1 = new Path("/test_file1"); - DFSTestUtil.createFile(fs, TEST_PATH1, 4096, - (short)1, 0xFADE1); - LOG.info("Setting failure injector and performing a read which " + - "should fail..."); - DataNodeFaultInjector failureInjector = Mockito.mock(DataNodeFaultInjector.class); - Mockito.doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - throw new IOException("injected error into sendShmResponse"); - } - }).when(failureInjector).sendShortCircuitShmResponse(); - DataNodeFaultInjector prevInjector = DataNodeFaultInjector.get(); - DataNodeFaultInjector.set(failureInjector); - try { - // The first read will try to allocate a shared memory segment and slot. - // The shared memory segment allocation will fail because of the failure - // injector. - DFSTestUtil.readFileBuffer(fs, TEST_PATH1); - fail("expected readFileBuffer to fail, but it succeeded."); - } catch (Throwable t) { - GenericTestUtils.assertExceptionContains("TCP reads were disabled for " + - "testing, but we failed to do a non-TCP read.", t); - } + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + final Path TEST_PATH1 = new Path("/test_file1"); + DFSTestUtil.createFile(fs, TEST_PATH1, 4096, + (short)1, 0xFADE1); + LOG.info("Setting failure injector and performing a read which " + + "should fail..."); + DataNodeFaultInjector failureInjector = Mockito.mock(DataNodeFaultInjector.class); + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + throw new IOException("injected error into sendShmResponse"); + } + }).when(failureInjector).sendShortCircuitShmResponse(); + DataNodeFaultInjector prevInjector = DataNodeFaultInjector.get(); + DataNodeFaultInjector.set(failureInjector); - checkNumberOfSegmentsAndSlots(0, 0, - cluster.getDataNodes().get(0).getShortCircuitRegistry()); + try { + // The first read will try to allocate a shared memory segment and slot. + // The shared memory segment allocation will fail because of the failure + // injector. + DFSTestUtil.readFileBuffer(fs, TEST_PATH1); + fail("expected readFileBuffer to fail, but it succeeded."); + } catch (Throwable t) { + GenericTestUtils.assertExceptionContains("TCP reads were disabled for " + + "testing, but we failed to do a non-TCP read.", t); + } - LOG.info("Clearing failure injector and performing another read..."); - DataNodeFaultInjector.set(prevInjector); + checkNumberOfSegmentsAndSlots(0, 0, + cluster.getDataNodes().get(0).getShortCircuitRegistry()); - fs.getClient().getClientContext().getDomainSocketFactory().clearPathMap(); + LOG.info("Clearing failure injector and performing another read..."); + DataNodeFaultInjector.set(prevInjector); - // The second read should succeed. - DFSTestUtil.readFileBuffer(fs, TEST_PATH1); + fs.getClient().getClientContext().getDomainSocketFactory().clearPathMap(); - // We should have added a new short-circuit shared memory segment and slot. - checkNumberOfSegmentsAndSlots(1, 1, - cluster.getDataNodes().get(0).getShortCircuitRegistry()); + // The second read should succeed. + DFSTestUtil.readFileBuffer(fs, TEST_PATH1); - cluster.shutdown(); - sockDir.close(); + // We should have added a new short-circuit shared memory segment and slot. + checkNumberOfSegmentsAndSlots(1, 1, + cluster.getDataNodes().get(0).getShortCircuitRegistry()); + } finally { + cluster.shutdown(); + sockDir.close(); + } } public static class TestPreReceiptVerificationFailureInjector @@ -824,20 +838,23 @@ public void testPreReceiptVerificationDfsClientCanDoScr() throws Exception { 1000000000L); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); - cluster.waitActive(); - DistributedFileSystem fs = cluster.getFileSystem(); - BlockReaderFactory.setFailureInjectorForTesting( - new TestPreReceiptVerificationFailureInjector()); - final Path TEST_PATH1 = new Path("/test_file1"); - DFSTestUtil.createFile(fs, TEST_PATH1, 4096, (short)1, 0xFADE2); - final Path TEST_PATH2 = new Path("/test_file2"); - DFSTestUtil.createFile(fs, TEST_PATH2, 4096, (short)1, 0xFADE2); - DFSTestUtil.readFileBuffer(fs, TEST_PATH1); - DFSTestUtil.readFileBuffer(fs, TEST_PATH2); - checkNumberOfSegmentsAndSlots(1, 2, - cluster.getDataNodes().get(0).getShortCircuitRegistry()); - cluster.shutdown(); - sockDir.close(); + try { + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + BlockReaderFactory.setFailureInjectorForTesting( + new TestPreReceiptVerificationFailureInjector()); + final Path TEST_PATH1 = new Path("/test_file1"); + DFSTestUtil.createFile(fs, TEST_PATH1, 4096, (short)1, 0xFADE2); + final Path TEST_PATH2 = new Path("/test_file2"); + DFSTestUtil.createFile(fs, TEST_PATH2, 4096, (short)1, 0xFADE2); + DFSTestUtil.readFileBuffer(fs, TEST_PATH1); + DFSTestUtil.readFileBuffer(fs, TEST_PATH2); + checkNumberOfSegmentsAndSlots(1, 2, + cluster.getDataNodes().get(0).getShortCircuitRegistry()); + } finally { + cluster.shutdown(); + sockDir.close(); + } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java index 50a401fe236c77..0c9695f4aec4ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java @@ -57,6 +57,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -80,6 +81,13 @@ public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest { } } + @AfterAll + public static void shutdownCluster() { + if (cluster != null) { + cluster.shutdown(); + } + } + @BeforeEach public void setUp() throws Exception { //get file system as a non-superuser From c9113d2270a3e1fae3e3e1e13864810629a2cbf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Luis=20L=C3=B3pez?= Date: Thu, 6 Aug 2026 07:57:48 +0200 Subject: [PATCH 3/4] HDFS-17957. HDFS test timeout review. Four mis-calibrated timeout budgets, and a per-method timeout default for hadoop-hdfs (600s) and hadoop-hdfs-rbf (1800s). Test-only. Contains content generated by Claude Code. Generated-by: Claude Code (Opus 5) Co-Authored-By: Claude Opus 5 --- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 9 +++++++++ .../router/TestRouterRPCClientRetries.java | 2 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 9 +++++++++ ...estBlockRecoveryCauseStandbyNameNodeCrash.java | 7 +++++-- .../balancer/TestBalancerWithHANameNodes.java | 15 ++++++--------- .../server/namenode/TestNameNodeRpcServer.java | 4 ++-- .../TestPersistentStoragePolicySatisfier.java | 2 +- .../namenode/ha/TestStandbyCheckpoints.java | 6 +++++- 8 files changed, 38 insertions(+), 16 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 556c4f3d175925..61f57e1aef4476 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -203,6 +203,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> ${project.build.directory}/derby.log + + 1800 s diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java index bdfb8165404a53..360fa3227bc070 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCClientRetries.java @@ -56,7 +56,7 @@ /** * Test retry behavior of the Router RPC Client. */ -@Timeout(100000) +@Timeout(100) public class TestRouterRPCClientRetries { private static StateStoreDFSCluster cluster; diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index e5a3cfa2d5fe91..64f8c6c6092a00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -266,6 +266,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> ${runningWithNative} + + 600 s diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java index 56fa0d49a01627..158ac0ee219afd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockRecoveryCauseStandbyNameNodeCrash.java @@ -73,9 +73,12 @@ public void setup() throws IOException { conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); final int numDNs = dataBlocks + parityBlocks; + // Ephemeral ports: the fixed-port variant simpleHATopology(2, 50070) + // binds 50070-50073 and fails with BindException whenever anything + // else on the CI agent holds one of them. cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numDNs) - .nnTopology(MiniDFSNNTopology.simpleHATopology(2, 50070)) + .nnTopology(MiniDFSNNTopology.simpleHATopology()) .build(); cluster.waitActive(); cluster.transitionToActive(0); @@ -156,7 +159,7 @@ public Boolean get() { return false; } } - }, 5000, 24000); + }, 5000, 60000); } catch (TimeoutException e) { throw new IOException("Timeout waiting for recoverLease()"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java index 1ebb475cf7708a..982cac70d5e288 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; -import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf; import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -106,13 +105,15 @@ public static void waitStoragesNoStale(MiniDFSCluster cluster, * datanodes); It then adds one new empty node and starts balancing. */ @Test - @Timeout(value = 60) + @Timeout(value = 300) public void testBalancerWithHANameNodes() throws Exception { + // 300s, not 60s: waitStoragesNoStale alone may legitimately take up + // to 60s, so a 60s test budget fires before the balancer even runs + // on a loaded CI agent. The observer variants below already use + // 120s/180s for the same work. Configuration conf = new HdfsConfiguration(); TestBalancer.initConf(conf); assertEquals(TEST_CAPACITIES.length, TEST_RACKS.length); - NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); - nn1Conf.setIpcPort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); Configuration copiedConf = new Configuration(conf); cluster = new MiniDFSCluster.Builder(copiedConf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) @@ -181,7 +182,7 @@ void doTest(Configuration conf, boolean withHA) throws Exception { * Test Balancer request Standby NameNode when enable this feature. */ @Test - @Timeout(value = 60) + @Timeout(value = 300) public void testBalancerRequestSBNWithHA() throws Exception { Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFS_NAMENODE_GETBLOCKS_CHECK_OPERATION_KEY, false); @@ -189,8 +190,6 @@ public void testBalancerRequestSBNWithHA() throws Exception { //conf.setBoolean(DFS_HA_BALANCER_REQUEST_STANDBY_KEY, true); TestBalancer.initConf(conf); assertEquals(TEST_CAPACITIES.length, TEST_RACKS.length); - NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); - nn1Conf.setIpcPort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); Configuration copiedConf = new Configuration(conf); cluster = new MiniDFSCluster.Builder(copiedConf) .nnTopology(MiniDFSNNTopology.simpleHATopology()) @@ -302,8 +301,6 @@ public void testGetLiveDatanodeStorageReport() throws Exception { Configuration conf = new HdfsConfiguration(); TestBalancer.initConf(conf); assertEquals(TEST_CAPACITIES.length, TEST_RACKS.length); - NNConf nn1Conf = new MiniDFSNNTopology.NNConf("nn1"); - nn1Conf.setIpcPort(HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); Configuration copiedConf = new Configuration(conf); // Try capture NameNodeConnector log. LogCapturer log =LogCapturer.captureLogs( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java index 7c203208ee4c63..70dffbb0c7dffe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java @@ -105,7 +105,7 @@ private static String getPreferredLocation(DistributedFileSystem fs, static final int ITERATIONS_TO_USE = 20; @Test - @Timeout(30000) + @Timeout(30) public void testNamenodeRpcClientIpProxyWithFailBack() throws Exception { // Make 3 nodes & racks so that we have a decent shot of detecting when // our change overrides the random choice of datanode. @@ -165,7 +165,7 @@ public void testNamenodeRpcClientIpProxyWithFailBack() throws Exception { } @Test - @Timeout(30000) + @Timeout(30) public void testObserverHandleAddBlock() throws Exception { String baseDir = GenericTestUtils.getRandomizedTempPath(); Configuration conf = new HdfsConfiguration(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java index eb52ff0a30c72b..823bcfaac76ddb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java @@ -318,7 +318,7 @@ public void testMultipleSatisfyStoragePolicy() throws Exception { * @throws Exception */ @Test - @Timeout(value = 300000) + @Timeout(value = 300) public void testDropSPS() throws Exception { try { clusterSetUp(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java index b22ed863ac1272..0fb31a80fbe479 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java @@ -423,8 +423,12 @@ public void testCheckpointCancellation() throws Exception { * mid-checkpoint during image upload from standby to active NN. */ @Test - @Timeout(value = 60) + @Timeout(value = 300) public void testCheckpointCancellationDuringUpload() throws Exception { + // 300s, not 60s: the internal waits alone (30s TransferFsImageUpload + // thread-death wait, two waitForStandbyToCatchUp/waitForCheckpoint + // rounds, three NameNode restarts, an upload throttled to 100 B/s) + // can exceed 60s on a loaded CI agent. Sibling tests use 300s. // Set dfs.namenode.checkpoint.txns differently on the first NN to avoid it // doing checkpoint when it becomes a standby cluster.getConfiguration(0).setInt( From 9b6c23132eb1053838a415d8304c53693b6c00b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Luis=20L=C3=B3pez?= Date: Fri, 14 Aug 2026 10:23:29 +0200 Subject: [PATCH 4/4] HDFS-17957. Fix indentation in the reindented test methods. Wrapping the bodies in try-with-resources shifted lines that already used a non-standard 4-space scheme, so checkstyle reported 45 new Indentation warnings. Reindent the affected methods to 2-space, and align the anonymous PrivilegedExceptionAction in TestDistributedFileSystem. Contains content generated by Claude Code. Generated-by: Claude Code (Opus 5) Co-Authored-By: Claude Opus 5 --- .../hdfs/TestDistributedFileSystem.java | 10 +- .../namenode/TestNetworkTopologyServlet.java | 152 +++++++++--------- 2 files changed, 81 insertions(+), 81 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 9d605eeef4f37e..089c1cffb2dd4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -1292,11 +1292,11 @@ public void testFileChecksum() throws Exception { System.out.println("webhdfsuri=" + webhdfsuri); final FileSystem webhdfs = ugi.doAs( new PrivilegedExceptionAction() { - @Override - public FileSystem run() throws Exception { - return new Path(webhdfsuri).getFileSystem(conf); - } - }); + @Override + public FileSystem run() throws Exception { + return new Path(webhdfsuri).getFileSystem(conf); + } + }); final Path dir = new Path("/filechecksum"); final int block_size = 1024; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java index 6bee0b275f8fe0..3a9119d350dfd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNetworkTopologyServlet.java @@ -89,55 +89,55 @@ public void testPrintTopologyTextFormat() throws IOException { @Test public void testPrintTopologyJsonFormat() throws IOException { - StaticMapping.resetMap(); - Configuration conf = new HdfsConfiguration(); - int dataNodesNum = 0; - final ArrayList rackList = new ArrayList(); - for (int i = 0; i < 5; i++) { - for (int j = 0; j < 2; j++) { - rackList.add("/rack" + i); - dataNodesNum++; - } + StaticMapping.resetMap(); + Configuration conf = new HdfsConfiguration(); + int dataNodesNum = 0; + final ArrayList rackList = new ArrayList(); + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 2; j++) { + rackList.add("/rack" + i); + dataNodesNum++; } + } + + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(dataNodesNum) + .racks(rackList.toArray(new String[rackList.size()])) + .build()) { + cluster.waitActive(); - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(dataNodesNum) - .racks(rackList.toArray(new String[rackList.size()])) - .build()) { - cluster.waitActive(); - - // get http uri - String httpUri = cluster.getHttpUri(0); - - // send http request - URL url = new URL(httpUri + "/topology"); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setReadTimeout(20000); - conn.setConnectTimeout(20000); - conn.setRequestProperty("Accept", "application/json"); - conn.connect(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); - String topology = out.toString(); - - // parse json - JsonNode racks = new ObjectMapper().readTree(topology); - - // assert rack number - assertEquals(racks.size(), 5); - - // assert node number - Iterator elements = racks.elements(); - int dataNodesCount = 0; - while(elements.hasNext()){ - JsonNode rack = elements.next(); - Iterator> fields = rack.fields(); - while (fields.hasNext()) { - dataNodesCount += fields.next().getValue().size(); - } - } - assertEquals(dataNodesCount, dataNodesNum); + // get http uri + String httpUri = cluster.getHttpUri(0); + + // send http request + URL url = new URL(httpUri + "/topology"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(20000); + conn.setConnectTimeout(20000); + conn.setRequestProperty("Accept", "application/json"); + conn.connect(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); + String topology = out.toString(); + + // parse json + JsonNode racks = new ObjectMapper().readTree(topology); + + // assert rack number + assertEquals(racks.size(), 5); + + // assert node number + Iterator elements = racks.elements(); + int dataNodesCount = 0; + while (elements.hasNext()) { + JsonNode rack = elements.next(); + Iterator> fields = rack.fields(); + while (fields.hasNext()) { + dataNodesCount += fields.next().getValue().size(); + } } + assertEquals(dataNodesCount, dataNodesNum); + } } @Test @@ -171,35 +171,35 @@ public void testPrintTopologyNoDatanodesTextFormat() throws IOException { } } - @Test - public void testPrintTopologyNoDatanodesJsonFormat() throws IOException { - StaticMapping.resetMap(); - Configuration conf = new HdfsConfiguration(); - try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) - .numDataNodes(0) - .build()) { - cluster.waitActive(); - - // get http uri - String httpUri = cluster.getHttpUri(0); - - // send http request - URL url = new URL(httpUri + "/topology"); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setReadTimeout(20000); - conn.setConnectTimeout(20000); - conn.setRequestProperty("Accept", "application/json"); - conn.connect(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); - StringBuilder sb = - new StringBuilder("-- Network Topology -- \n"); - sb.append(out); - sb.append("\n-- Network Topology -- "); - String topology = sb.toString(); - - // assert node number - assertTrue(topology.contains("No DataNodes")); - } + @Test + public void testPrintTopologyNoDatanodesJsonFormat() throws IOException { + StaticMapping.resetMap(); + Configuration conf = new HdfsConfiguration(); + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(0) + .build()) { + cluster.waitActive(); + + // get http uri + String httpUri = cluster.getHttpUri(0); + + // send http request + URL url = new URL(httpUri + "/topology"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setReadTimeout(20000); + conn.setConnectTimeout(20000); + conn.setRequestProperty("Accept", "application/json"); + conn.connect(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); + StringBuilder sb = + new StringBuilder("-- Network Topology -- \n"); + sb.append(out); + sb.append("\n-- Network Topology -- "); + String topology = sb.toString(); + + // assert node number + assertTrue(topology.contains("No DataNodes")); } + } }