Hadoop中的MapReduce程序，它实现了一个简单的“您可能认识的人”

Question

输入文件包含邻接列表,并具有以下格式的多行：

import java.io.IOException; 
import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.IntWritable; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapreduce.Job; 
import org.apache.hadoop.mapreduce.Mapper; 
import org.apache.hadoop.mapreduce.Reducer; 
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Friends 
{

public class FriendsMap extends Mapper < LongWritable, Text, Text, IntWritable >
    {
    private Text friendsAB;
    private Text friendsBA;
    private IntWritable one = new IntWritable(1);
    private IntWritable oneLess = new IntWritable(-999999999);
        //@SuppressWarnings("null")
        @Override 
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 
        { 
            String friendsOfA[] = null;     //This will be all of the friends of the user in this row
            String oneRow[] = value.toString().split("\t,");    //Break the row up into users IDs
            String userA = oneRow[0];       //This is the main user for this row
            for (int i=1; i < oneRow.length; i++)   //Create an array of the rest of the users in this row
            {
                friendsOfA[i-1] = oneRow[i];
            }
            for (int i=0; i < oneRow.length; i++)   //Output the main user in pairs with all friends plus a lagre negative #
            {
                friendsAB.set(userA + " " + friendsOfA[i]);
                context.write(friendsAB, oneLess);
                System.out.println(friendsAB + " " + oneLess);
            }
            for (int i = 0; i < friendsOfA.length; i++)     //Output each friend pair plus the number 1
            {
                for (int j = i + 1; j < friendsOfA.length; j++) 
                {
                    friendsAB.set(friendsOfA[i] + " " + friendsOfA[j]);
                    friendsBA.set(friendsOfA[j] + " " + friendsOfA[i]);
                    context.write(friendsAB, one);
                    context.write(friendsBA, one);
                    System.out.println(friendsAB + " " + one);
                    System.out.println(friendsBA + " " + one);
                }
            }
        }
    }

class FriendReducer extends Reducer < Text, IntWritable, Text, IntWritable > 
    { 
        private IntWritable result = new IntWritable(); 
        @Override 
        public void reduce( Text key, Iterable < IntWritable > values, Context context) throws IOException, InterruptedException 
        { 
            int sum = 0; 
            for (IntWritable val : values) 
            { 
                sum += val.get(); 
            } 
            if (sum > 1)
            {
                result.set( sum); 
                context.write( key, result);
            }
            //At this point I have all pairs of users with recomenede friends and a count of how many times they each
            //friend has been recomended to a user.
            //I need to sort by user and then by number of recomendations.
            //Then print the user <tab> all recomendations with commas between them.
        } 
    }


public static void main( String[] args) throws Exception 
    { 
        Configuration conf = new Configuration();
        Job job = Job.getInstance( conf, "Friends");
        job.setJarByClass(Friends.class);
        FileInputFormat.addInputPath( job, new Path("input")); 
        FileOutputFormat.setOutputPath( job, new Path("output")); 
        job.setMapperClass( FriendsMap.class); 
        job.setCombinerClass( FriendReducer.class); 
        job.setReducerClass( FriendReducer.class);
        job.setOutputKeyClass( Text.class); 
        job.setOutputValueClass( IntWritable.class);

        System.exit( job.waitForCompletion( true) ? 0 : 1); 
    }
}

将类更改为静态后，这就是新的错误。

我觉得这是最麻烦的部分。

某些错误的屏幕快照

public static class FriendsMap extends Mapper < LongWritable, Text, Text, IntWritable >
    {
        //@SuppressWarnings("null")
        @Override 
        public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 
        { 

            String friendsOfA[];    //This will be all of the friends of the user in this row
            friendsOfA = new String[] {};
            String friendsAB        = "1";      //This will be used to create pairs of users
            String friendsBA        = "2";      //This will be used to create pairs of users
            Text pairA;
            Text pairB; 
            IntWritable one     = new IntWritable(1);           //1 if they are not an existing pair here
            IntWritable oneLess = new IntWritable(-999999999);  // if they are an existing pair

            String oneRow[] = value.toString().split("\t,");    //Break the row up into users IDs
            Text userA = new Text(oneRow[0]);                                   //This is the main user for this row
            for (int i=1; i < oneRow.length; i++)   //Create an array of the rest of the users in this row
            {
                friendsOfA[i-1] = oneRow[i];
            }
            for (int i=0; i < oneRow.length; i++)   //Output the main user in pairs with all friends plus a large negative #
            {                                       //We do not want to recommend them as friends because they are friends 
                Text FOA = new Text (friendsOfA[i]);
                friendsAB = (userA + " " + FOA);
                Text pair = new Text (friendsAB);
                context.write(pair, oneLess);
                System.out.println(pair + " " + oneLess);
            }
            for (int i = 0; i < friendsOfA.length; i++)     //Output each friend pair plus the number 1
            {                                               //We want to recommend them as potential friends
                for (int j = i + 1; j < friendsOfA.length; j++) 
                {
                    Text FOA = new Text (friendsOfA[i]);
                    Text FOB = new Text (friendsOfA[j]);
                    friendsAB = (FOA + " " + FOB);
                    friendsBA = (FOB + " " + FOA);
                    pairA = new Text (friendsAB);
                    pairB = new Text (friendsBA);
                    context.write(pairA, one);
                    context.write(pairB, one);
                    System.out.println(pairA + " " + one);
                    System.out.println(pairB + " " + one);
                }
            }
        }
    }

Answer 1

您已经将类作为内部类进行了处理，这可能会引起一些问题。内部类只能存在于封闭类的实例中。

将它们更改为静态类可能更容易。

public class Friends {

    public static class FriendsMap extends Mapper <...> {}

    public static class FriendReducer extends Reducer <...> {}

    public static void main( String[] args) throws Exception { 
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "Friends");
        job.setJarByClass(Friends.class);
        FileInputFormat.addInputPath(job, new Path("input")); 
        FileOutputFormat.setOutputPath(job, new Path("output")); 
        job.setMapperClass(FriendsMap.class); 
        job.setCombinerClass(FriendReducer.class); 
        job.setReducerClass(FriendReducer.class);
        job.setOutputKeyClass(Text.class); 
        job.setOutputValueClass(IntWritable.class);

        System.exit( job.waitForCompletion( true) ? 0 : 1); 
    }
}