TestSimulatedFSDataset.java
上传用户:quxuerui
上传日期:2018-01-08
资源大小:41811k
文件大小:10k
- /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package org.apache.hadoop.hdfs.server.datanode;
- import java.io.DataInputStream;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.OutputStream;
- import junit.framework.TestCase;
- import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.hdfs.protocol.Block;
- import org.apache.hadoop.hdfs.server.datanode.FSDatasetInterface;
- import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
- import org.apache.hadoop.util.DataChecksum;
- /**
- * this class tests the methods of the SimulatedFSDataset.
- *
- */
- public class TestSimulatedFSDataset extends TestCase {
-
- Configuration conf = null;
-
-
- static final int NUMBLOCKS = 20;
- static final int BLOCK_LENGTH_MULTIPLIER = 79;
- protected void setUp() throws Exception {
- super.setUp();
- conf = new Configuration();
- conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
-
- }
- protected void tearDown() throws Exception {
- super.tearDown();
- }
-
- long blockIdToLen(long blkid) {
- return blkid*BLOCK_LENGTH_MULTIPLIER;
- }
-
- int addSomeBlocks(FSDatasetInterface fsdataset, int startingBlockId) throws IOException {
- int bytesAdded = 0;
- for (int i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) {
- Block b = new Block(i, 0, 0); // we pass expected len as zero, - fsdataset should use the sizeof actual data written
- OutputStream dataOut = fsdataset.writeToBlock(b, false).dataOut;
- assertEquals(0, fsdataset.getLength(b));
- for (int j=1; j <= blockIdToLen(i); ++j) {
- dataOut.write(j);
- assertEquals(j, fsdataset.getLength(b)); // correct length even as we write
- bytesAdded++;
- }
- dataOut.close();
- b.setNumBytes(blockIdToLen(i));
- fsdataset.finalizeBlock(b);
- assertEquals(blockIdToLen(i), fsdataset.getLength(b));
- }
- return bytesAdded;
- }
- int addSomeBlocks(FSDatasetInterface fsdataset ) throws IOException {
- return addSomeBlocks(fsdataset, 1);
- }
- public void testGetMetaData() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- Block b = new Block(1, 5, 0);
- try {
- assertFalse(fsdataset.metaFileExists(b));
- assertTrue("Expected an IO exception", false);
- } catch (IOException e) {
- // ok - as expected
- }
- addSomeBlocks(fsdataset); // Only need to add one but ....
- b = new Block(1, 0, 0);
- InputStream metaInput = fsdataset.getMetaDataInputStream(b);
- DataInputStream metaDataInput = new DataInputStream(metaInput);
- short version = metaDataInput.readShort();
- assertEquals(FSDataset.METADATA_VERSION, version);
- DataChecksum checksum = DataChecksum.newDataChecksum(metaDataInput);
- assertEquals(DataChecksum.CHECKSUM_NULL, checksum.getChecksumType());
- assertEquals(0, checksum.getChecksumSize());
- }
- public void testStorageUsage() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- assertEquals(fsdataset.getDfsUsed(), 0);
- assertEquals(fsdataset.getRemaining(), fsdataset.getCapacity());
- int bytesAdded = addSomeBlocks(fsdataset);
- assertEquals(bytesAdded, fsdataset.getDfsUsed());
- assertEquals(fsdataset.getCapacity()-bytesAdded, fsdataset.getRemaining());
-
- }
- void checkBlockDataAndSize(FSDatasetInterface fsdataset,
- Block b, long expectedLen) throws IOException {
- InputStream input = fsdataset.getBlockInputStream(b);
- long lengthRead = 0;
- int data;
- while ((data = input.read()) != -1) {
- assertEquals(SimulatedFSDataset.DEFAULT_DATABYTE, data);
- lengthRead++;
- }
- assertEquals(expectedLen, lengthRead);
- }
-
- public void testWriteRead() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- addSomeBlocks(fsdataset);
- for (int i=1; i <= NUMBLOCKS; ++i) {
- Block b = new Block(i, 0, 0);
- assertTrue(fsdataset.isValidBlock(b));
- assertEquals(blockIdToLen(i), fsdataset.getLength(b));
- checkBlockDataAndSize(fsdataset, b, blockIdToLen(i));
- }
- }
- public void testGetBlockReport() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- Block[] blockReport = fsdataset.getBlockReport();
- assertEquals(0, blockReport.length);
- int bytesAdded = addSomeBlocks(fsdataset);
- blockReport = fsdataset.getBlockReport();
- assertEquals(NUMBLOCKS, blockReport.length);
- for (Block b: blockReport) {
- assertNotNull(b);
- assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
- }
- }
- public void testInjectionEmpty() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- Block[] blockReport = fsdataset.getBlockReport();
- assertEquals(0, blockReport.length);
- int bytesAdded = addSomeBlocks(fsdataset);
- blockReport = fsdataset.getBlockReport();
- assertEquals(NUMBLOCKS, blockReport.length);
- for (Block b: blockReport) {
- assertNotNull(b);
- assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
- }
-
- // Inject blocks into an empty fsdataset
- // - injecting the blocks we got above.
-
-
- SimulatedFSDataset sfsdataset = new SimulatedFSDataset(conf);
- sfsdataset.injectBlocks(blockReport);
- blockReport = sfsdataset.getBlockReport();
- assertEquals(NUMBLOCKS, blockReport.length);
- for (Block b: blockReport) {
- assertNotNull(b);
- assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
- assertEquals(blockIdToLen(b.getBlockId()), sfsdataset.getLength(b));
- }
- assertEquals(bytesAdded, sfsdataset.getDfsUsed());
- assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
- }
- public void testInjectionNonEmpty() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
-
- Block[] blockReport = fsdataset.getBlockReport();
- assertEquals(0, blockReport.length);
- int bytesAdded = addSomeBlocks(fsdataset);
- blockReport = fsdataset.getBlockReport();
- assertEquals(NUMBLOCKS, blockReport.length);
- for (Block b: blockReport) {
- assertNotNull(b);
- assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
- }
- fsdataset = null;
-
- // Inject blocks into an non-empty fsdataset
- // - injecting the blocks we got above.
-
-
- SimulatedFSDataset sfsdataset = new SimulatedFSDataset(conf);
- // Add come blocks whose block ids do not conflict with
- // the ones we are going to inject.
- bytesAdded += addSomeBlocks(sfsdataset, NUMBLOCKS+1);
- Block[] blockReport2 = sfsdataset.getBlockReport();
- assertEquals(NUMBLOCKS, blockReport.length);
- blockReport2 = sfsdataset.getBlockReport();
- assertEquals(NUMBLOCKS, blockReport.length);
- sfsdataset.injectBlocks(blockReport);
- blockReport = sfsdataset.getBlockReport();
- assertEquals(NUMBLOCKS*2, blockReport.length);
- for (Block b: blockReport) {
- assertNotNull(b);
- assertEquals(blockIdToLen(b.getBlockId()), b.getNumBytes());
- assertEquals(blockIdToLen(b.getBlockId()), sfsdataset.getLength(b));
- }
- assertEquals(bytesAdded, sfsdataset.getDfsUsed());
- assertEquals(sfsdataset.getCapacity()-bytesAdded, sfsdataset.getRemaining());
-
-
- // Now test that the dataset cannot be created if it does not have sufficient cap
- conf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY, 10);
-
- try {
- sfsdataset = new SimulatedFSDataset(conf);
- sfsdataset.injectBlocks(blockReport);
- assertTrue("Expected an IO exception", false);
- } catch (IOException e) {
- // ok - as expected
- }
- }
- public void checkInvalidBlock(Block b) throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- assertFalse(fsdataset.isValidBlock(b));
- try {
- fsdataset.getLength(b);
- assertTrue("Expected an IO exception", false);
- } catch (IOException e) {
- // ok - as expected
- }
-
- try {
- fsdataset.getBlockInputStream(b);
- assertTrue("Expected an IO exception", false);
- } catch (IOException e) {
- // ok - as expected
- }
-
- try {
- fsdataset.finalizeBlock(b);
- assertTrue("Expected an IO exception", false);
- } catch (IOException e) {
- // ok - as expected
- }
-
- }
-
- public void testInValidBlocks() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- Block b = new Block(1, 5, 0);
- checkInvalidBlock(b);
-
- // Now check invlaid after adding some blocks
- addSomeBlocks(fsdataset);
- b = new Block(NUMBLOCKS + 99, 5, 0);
- checkInvalidBlock(b);
-
- }
- public void testInvalidate() throws IOException {
- FSDatasetInterface fsdataset = new SimulatedFSDataset(conf);
- int bytesAdded = addSomeBlocks(fsdataset);
- Block[] deleteBlocks = new Block[2];
- deleteBlocks[0] = new Block(1, 0, 0);
- deleteBlocks[1] = new Block(2, 0, 0);
- fsdataset.invalidate(deleteBlocks);
- checkInvalidBlock(deleteBlocks[0]);
- checkInvalidBlock(deleteBlocks[1]);
- long sizeDeleted = blockIdToLen(1) + blockIdToLen(2);
- assertEquals(bytesAdded-sizeDeleted, fsdataset.getDfsUsed());
- assertEquals(fsdataset.getCapacity()-bytesAdded+sizeDeleted, fsdataset.getRemaining());
-
-
-
- // Now make sure the rest of the blocks are valid
- for (int i=3; i <= NUMBLOCKS; ++i) {
- Block b = new Block(i, 0, 0);
- assertTrue(fsdataset.isValidBlock(b));
- }
- }
- }