在HDP 2.0上运行hadoop map reduce java程序时出错

Question

我在java中运行mapreduce程序，它编译成功，但在运行时我使用的是EC2上的Hortonworks HDP2.0。

    package edu.osu.cse.ysmart.TestQuery;
    import java.io.IOException;
    import java.util.*;
    import java.text.*;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.conf.*;
    import org.apache.hadoop.io.*;
    import org.apache.hadoop.util.Tool;
    import org.apache.hadoop.util.ToolRunner;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.Mapper;
    import org.apache.hadoop.mapreduce.Reducer;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.input.FileSplit;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
    import org.apache.hadoop.mapreduce.lib.partition.*;


    public class TestQuery1 extends Configured implements Tool{

            public static class Map extends Mapper<Object, Text,NullWritable,Text>{

                    public void map(Object key, Text value, Context context) throws IOException,InterruptedException{

                            String line = value.toString();
                            String[] line_buf = new String[3];
                            int prev=0,i=0,n=0;
                            for(i=0,n=0,prev=0;i<line.length();i++){

                                    if (line.charAt(i) == '|'){
                                            line_buf[n] = line.substring(prev,i);
                                            n = n+1;
                                            prev = i+1;
                                    }
                                    if(n == 3)
                                            break;
                            }

                            if(n<3)
                                    line_buf[n] = line.substring(prev,i);
                            if(line_buf[2].compareTo("Abid Khan") == 0){

                                    NullWritable key_op = NullWritable.get();
                                    context.write(key_op , new Text(line_buf[1]+ "|" ));
                            }
                    }

            }

            public int run(String[] args) throws Exception{

                    Configuration conf = new Configuration();
                    Job job = new Job(conf,"TestQuery1");
                    job.setJarByClass(TestQuery1.class);
                    job.setMapOutputKeyClass(NullWritable.class);
                    job.setMapOutputValueClass(Text.class);
                    job.setOutputKeyClass(NullWritable.class);
                    job.setOutputValueClass(Text.class);
                    job.setMapperClass(Map.class);
                    FileInputFormat.addInputPath(job, new Path(args[0]));
                    FileOutputFormat.setOutputPath(job, new Path(args[1]));
                    return (job.waitForCompletion(true) ? 0 : 1);
            }

            public static void main(String[] args) throws Exception {

                            int res = ToolRunner.run(new Configuration(), new TestQuery1(), args);
                            System.exit(res);
            }

    }

好心的帮助

Answer 1

从异常跟踪中可以看出,您试图在空对象上调用一个方法：

attempt_1388768204803_0006_m_000000_0, Status : FAILED Error: java.lang.NullPointerException at edu.osu.cse.ysmart.TestQuery.TestQuery1$Map.map(TestQuery1.java:42) at 

从源代码来看，以下比较失败：

if(line_buf[2].compareTo("Abid Khan") == 0)

未填充line_buf[2]，因此出现空指针异常。您可以在if条件之前打印line_buf[2]的值，以查看是否有值。这将帮助您调试问题。