pom
<dependency>
<groupId>org.apache.mrunit</groupId>
<artifactId>mrunit</artifactId>
<version>1.1.0</version>
<!--<scope>test</scope>-->
<!--不加导包可能失败-->
<classifier>hadoop2</classifier>
</dependency>
以《Hadoop权威指南》上的MaxTemperature程序为例,整个项目中包括如下4个源文件,前两个分别是Mapper程序和Reducer程序,后两个分别是针对Mapper和Reducer的测试程序:
MaxTemperatureMapper.java MaxTemperatureReducer.java MaxTemperatureMapperTest.java MaxTemperatureReducerTest.java |
MaxTemperatureMapper.java
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class MaxTemperatureMapper
extends Mapper<LongWritable, Text, Text, IntWritable> {
private static final int MISSING = 9999;
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String year = line.substring(15, 19);
int airTemperature;
if(line.charAt(87) == '+')
airTemperature = Integer.parseInt(line.substring(88, 92));
else
airTemperature = Integer.parseInt(line.substring(87, 92));
String quality = line.substring(92, 93);
if(airTemperature != MISSING && quality.matches("[01459]"))
context.write(new Text(year), new IntWritable(airTemperature));
}
}
MaxTemperatureReducer.java
import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class MaxTemperatureReducer
extends Reducer<Text, IntWritable, Text, IntWritable> {
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int maxValue = Integer.MIN_VALUE;
for(IntWritable val : values)
maxValue = Math.max(maxValue, val.get());
context.write(key, new IntWritable(maxValue));
}
}
MaxTemperatureMapperTest.java
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.junit.Test;
public class MaxTemperatureMapperTest {
@Test
public void processesValidRecord() throws IOException, InterruptedException {
Text value = new Text("0057332130999991950010103004+51317+028783FM-12+017199999V0203201N00721004501CN0100001N9-01281-01391102681");
new MapDriver<LongWritable, Text, Text, IntWritable>()
.withMapper(new MaxTemperatureMapper())
.withInput(new LongWritable(0), value)
.withOutput(new Text("1950"), new IntWritable(-128))
.runTest();
}
}
MaxTemperatureReducerTest.java
import java.io.IOException;
import java.util.Arrays;
import org.junit.Test;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
public class MaxTemperatureReducerTest {
@Test
public void returnsMaximumIntegerInValues() throws IOException, InterruptedException {
new ReduceDriver<Text, IntWritable, Text, IntWritable>()
.withReducer(new MaxTemperatureReducer())
.withInput(new Text("1950"), Arrays.asList(new IntWritable(10), new IntWritable(5)))
.withOutput(new Text("1950"), new IntWritable(10))
.runTest();
}
}
需要注意程序中的@Test是不可或缺的
下面就可以运行测试程序了,如果要测试Mapper程序,那么就选中MaxTemperatureMapperTest.java,右键Run As-->JUnit Test,如果进度条为绿色,则表示测试正确,否则代表有错误。测试Reducer程序也是如此