如何使用WritableComparator Hadoop

Question

下面是我使用WritableComparator的代码段，但它不起作用

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

public class MovieComparator extends WritableComparator{

    public MovieComparator(){
        super(Movie.class);
    }
    @Override
    public int compare(WritableComparable o,WritableComparable o2){
        System.out.println("in compare");
        Movie m = (Movie)o;
        Movie m2 = (Movie)o2;
        System.out.println(m.compareTo(m2));
        return m.movieId.compareTo(m2.movieId);
    }

}

public class Movie implements WritableComparable {

    Text movieId;
    Text movieTitle;

    public Movie(Text movieId, Text movieTitle) {

        this.movieId = movieId;
        this.movieTitle = movieTitle;
    }

    public Movie(){

    }

    public String getMovieId() {
        return movieId.toString();
    }

    public void setMovieId(String movieId) {
        this.movieId = new Text(movieId);
    }

    public String getMovieTitle() {
        return movieTitle.toString();
    }

    public void setMovieTitle(String movieTitle) {
        this.movieTitle = new Text(movieTitle);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        //movieId = in.read;
        movieId.readFields(in);
        movieTitle.readFields(in);
    }

    @Override
    public void write(DataOutput out) throws IOException {
        //out.writeUTF(movieId);
        //out.writeUTF(movieTitle);
        movieId.write(out);
        movieTitle.write(out);

    }

    @Override
    public int compareTo(Movie o) {

    //  System.out.println("in compareTo");
        int res=movieTitle.compareTo(o.movieTitle);
        return res;
    }



    @Override
    public int hashCode(){
        return movieId.hashCode();
    }

    @Override
    public boolean equals(Object o){
        Movie m=(Movie)o; 
        return movieId.equals(m.movieId);
    }

    @Override
    public String toString(){
        return movieTitle.toString();
    }

}

In driver class I am setting the comparator by below line
job.setSortComparatorClass(MovieComparator.class);
14/09/08 14:17:03 WARN mapred.LocalJobRunner: job_local_0001
java.io.IOException: Spill failed
    at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1029)
    at org.apache.hadoop.mapred.MapTask$NewOutputCollector.write(MapTask.java:691)
    at org.apache.hadoop.mapreduce.TaskInputOutputContext.write(TaskInputOutputContext.java:80)
    at com.impetus.MovieMapper.map(MovieMapper.java:44)
    at com.impetus.MovieMapper.map(MovieMapper.java:1)
    at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:144)
    at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:764)
    at org.apache.hadoop.mapred.MapTask.run(MapTask.java:370)
    at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:212)
                

Answer 1

我发现了一个问题，即不能使用super(movie.class)，而必须使用super(movie.class，true)。通过发送true，WritableComparator将实例化对象，否则它将在compare方法中传递null