TestSequenceFileMergeProgress.java
上传用户:quxuerui
上传日期:2018-01-08
资源大小:41811k
文件大小:3k
源码类别:
网格计算
开发平台:
Java
- /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package org.apache.hadoop.io;
- import java.io.IOException;
- import java.util.ArrayList;
- import java.util.List;
- import java.util.Random;
- import org.apache.hadoop.fs.*;
- import org.apache.hadoop.io.*;
- import org.apache.hadoop.io.SequenceFile.CompressionType;
- import org.apache.hadoop.io.SequenceFile.Sorter.RawKeyValueIterator;
- import org.apache.hadoop.io.SequenceFile.Sorter.SegmentDescriptor;
- import org.apache.hadoop.io.compress.CompressionCodec;
- import org.apache.hadoop.io.compress.DefaultCodec;
- import org.apache.hadoop.mapred.*;
- import junit.framework.TestCase;
- import org.apache.commons.logging.*;
- public class TestSequenceFileMergeProgress extends TestCase {
- private static final Log LOG = FileInputFormat.LOG;
- private static final int RECORDS = 10000;
- public void testMergeProgressWithNoCompression() throws IOException {
- runTest(SequenceFile.CompressionType.NONE);
- }
- public void testMergeProgressWithRecordCompression() throws IOException {
- runTest(SequenceFile.CompressionType.RECORD);
- }
- public void testMergeProgressWithBlockCompression() throws IOException {
- runTest(SequenceFile.CompressionType.BLOCK);
- }
- public void runTest(CompressionType compressionType) throws IOException {
- JobConf job = new JobConf();
- FileSystem fs = FileSystem.getLocal(job);
- Path dir = new Path(System.getProperty("test.build.data",".") + "/mapred");
- Path file = new Path(dir, "test.seq");
- Path tempDir = new Path(dir, "tmp");
- fs.delete(dir, true);
- FileInputFormat.setInputPaths(job, dir);
- fs.mkdirs(tempDir);
- LongWritable tkey = new LongWritable();
- Text tval = new Text();
- SequenceFile.Writer writer =
- SequenceFile.createWriter(fs, job, file, LongWritable.class, Text.class,
- compressionType, new DefaultCodec());
- try {
- for (int i = 0; i < RECORDS; ++i) {
- tkey.set(1234);
- tval.set("valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue");
- writer.append(tkey, tval);
- }
- } finally {
- writer.close();
- }
- long fileLength = fs.getFileStatus(file).getLen();
- LOG.info("With compression = " + compressionType + ": "
- + "compressed length = " + fileLength);
- SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
- job.getOutputKeyComparator(), job.getMapOutputKeyClass(),
- job.getMapOutputValueClass(), job);
- Path[] paths = new Path[] {file};
- RawKeyValueIterator rIter = sorter.merge(paths, tempDir, false);
- int count = 0;
- while (rIter.next()) {
- count++;
- }
- assertEquals(RECORDS, count);
- assertEquals(1.0f, rIter.getProgress().get());
- }
- }