这个Java for HDInsight Hadoop有什么问题吗？

Question

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;

public class PageStat implements Tool {
private Configuration conf;

@Override
public int run(String[] args) throws Exception {
    Job job = new Job(getConf());
    String jobName = "Page visit statistics MR";
    job.setJobName(jobName);

    job.setJarByClass(PageStat.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(PageStat.PageStatMapper.class);
    job.setReducerClass(PageStat.PageStatReducer.class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setNumReduceTasks(job.getConfiguration().getInt("num.reducer", 1));

    int status =  job.waitForCompletion(true) ? 0 : 1;
    return status;
}

public static void main(String[] args) throws Exception {
    int exitCode = ToolRunner.run(new PageStat(), args);
    System.exit(exitCode);
}

public void setConf(Configuration conf) {
   this.conf = conf;
}

public Configuration getConf() {
    return conf;
}

public static class PageStatMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
    private Text keyHolder = new Text();
    private IntWritable valueHolder = new IntWritable();

    @Override
    protected void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {
        String[] items  =  value.toString().split(",");
        if (items.length == 3) {
            String url = items[1];
            keyHolder.set(url);
            Integer duration = Integer.parseInt(items[2]);
            valueHolder.set(duration);
            context.write(keyHolder, valueHolder);
        } else {
            context.getCounter("Error", "invalidData").increment(1);
        }
    }        
}   

public static class PageStatReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
    private Text keyHolder = new Text();
    private IntWritable valueHolder = new IntWritable();
    private String statType;
    private int count;
    private int totalTime;
    private int avTime;

    protected void setup(Context context) throws IOException, InterruptedException {
        Configuration conf = context.getConfiguration();
        statType = conf.get("page.stat");
    }

    protected void reduce(Text key, Iterable<IntWritable> values, Context context)
    throws IOException, InterruptedException {
        count = 0;
        totalTime = 0;
        for (IntWritable value : values){
            ++count;
            totalTime += value.get();
        } 
        avTime = totalTime / count;

        keyHolder.set(key);
        if (statType.equals("average")){
            valueHolder.set(avTime);
        } else {
            valueHolder.set(totalTime);
        }
        context.write(keyHolder, valueHolder);
    }
}    

}

谢谢！！！

Answer 1

我遇到了一个类似的问题，您需要使用-d标志来执行：

-Dpage.stat=total

您可能会看到一个错误：

log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
log4j:WARN Please initialize the log4j system properly.

这还不是全部答案，我自己还在查个水落石出。