为简单的hadoop mapreduce作业运行两个mapper和两个reducer

Question

我一次就完成了这项工作。我以

$hadoop jar job.jar输入输出

我已经开始了

$ hadoop namenode -format
$ hadoop namenode

$ hadoop datanode

package org.apache.hadoop.examples;

import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
import org.rg.apache.hadoop.fs.Path;
import oapache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class WordCount {
private static final Log LOG = LogFactory.getLog(WordCount.class);

  public static class TokenizerMapper
       extends Mapper<Object, Text, Text, IntWritable>{

    private final static IntWritable one = new IntWritable(1);
    private Text word = new Text();

    public void map(Object key, Text value, Context context
                    ) throws IOException, InterruptedException {
      StringTokenizer itr = new StringTokenizer(value.toString());
      while (itr.hasMoreTokens()) {
        word.set(itr.nextToken());
        context.write(word, one);
      }
    }
  }

  public static class IntSumReducer
       extends Reducer<Text,IntWritable,Text,IntWritable> {
    private IntWritable result = new IntWritable();

    public void reduce(Text key, Iterable<IntWritable> values,
                       Context context
                       ) throws IOException, InterruptedException {
      int sum = 0;
      //printKeyAndValues(key, values);

      for (IntWritable val : values) {
        sum += val.get();
      LOG.info("val = " + val.get());
      }
      LOG.info("sum = " + sum + " key = " + key);
      result.set(sum);
      context.write(key, result);
      //System.err.println(String.format("[reduce] word: (%s), count: (%d)", key, result.get()));
    }


  // a little method to print debug output
    private void printKeyAndValues(Text key, Iterable<IntWritable> values)
    {
      StringBuilder sb = new StringBuilder();
      for (IntWritable val : values)
      {
        sb.append(val.get() + ", ");
      }
      System.err.println(String.format("[reduce] key: (%s), value: (%s)", key, sb.toString()));
    }
  }

  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
      System.err.println("Usage: wordcount <in> <out>");
      System.exit(2);
    }
    Job job = new Job(conf, "word count");
    job.setJarByClass(WordCount.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}

现在你们谁能帮我运行两个映射器和简化器来完成这个单词计数工作吗？

Answer 1

Gladnick：如果您打算使用默认的TextInputFormat格式，那么在输入文件的数量上至少会有同样多的映射器（或者更多取决于文件的大小）。因此，只需将2个文件放入您的输入目录，这样您就可以运行2个映射器了。（建议使用此解决方案，因为您计划将其作为测试用例运行）。

既然您已经要求了2个减速器，那么您所需要做的就是job.SetNumReduceTasks(2)在您的主要befor submiting the job。

之后，只需准备一个应用程序的jar并在hadoop伪集群中运行它。

            Configuration configuration = new Configuration();
        // create a configuration object that provides access to various
        // configuration parameters
        Job job = new Job(configuration, "Wordcount-Vowels & Consonants");
        // create the job object and set job name as Wordcount-Vowels &
        // Consonants
        job.setJarByClass(WordCount.class);
        // set the main class
        job.setNumReduceTasks(2);
        // set the number of reduce tasks required
        job.setMapperClass(WordCountMapper.class);
        // set the map class for the job
        job.setCombinerClass(WordCountCombiner.class);
        // set the combiner class for the job
        job.setPartitionerClass(VowelConsonantPartitioner.class);
        // set the partitioner class for the job
        job.setReducerClass(WordCountReducer.class);
        // set the reduce class for the job
        job.setOutputKeyClass(Text.class);
        // set the output type of key (the word) expected from the job, Text
        // analogous to String
        job.setOutputValueClass(IntWritable.class);
        // set the output type of value (the count) expected from the job,
        // IntWritable analogous to int
        FileInputFormat.addInputPath(job, new Path(args[0]));
        // set the input directory for fetching the input files
        FileOutputFormat.setOutputPath(job, new Path(args[1]));