MapReduce編程重點把握

MapReduce核心概念

    中心思想：分而治之，分布式計算模型      
    Map（映射）      
    分布式的計算模型，處理海量數據      
    一個簡單的MR程序需要指定：      
    map()、reduce()、input、output      
    處理的數據放在input中，處理的結果放在output中      ![](http://upload-images.jianshu.io/upload_images/2409103-c893fe48912e145a.jpg?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240)      
    MR有一定的格式如下：

以<key, value>進行流向的
input -> map() -> reduce() -> output

思考幾個問題

    1. 對處理的文件轉化成什么樣的<key, value>？      
    2. map()輸出結果變成什么樣的<key, value>？      
    3. reduce()是怎么處理<key, value>的，輸出的<key, value>又是什么樣的？

詞頻統計wordcount的具體執行過程

默認是把每一行當成一個<key, value>
hadoop hdfs -> <0, hadoop hdfs>
hadoop mapreduce hadoop yarn
hadoop hello
mapreduce hadoop
yarn hadoop
比如我們要統計單詞（1）分割單詞，按照空格進行分詞
hadoop hdfs -> hadoop hdfs
記錄出現的次數，每出現一次記錄為：
Map() <hadoop, 1> <mapreduce, 1>
<hadoop, 1> <yarn, 1>
reduce的輸入就是map的輸出
reduce() 將相同key的value累加到一起
<hadoop, list(1,1)>
<mapreduce, list(1)>
<hdfs, list(1)>
方法的重寫shift+Alt+S

MapReduce代碼邏輯主干初寫

//Step 3: Driver public int run(String[] args) throws Exception {
//read configuration file
Configuration configuration = new Configuration();
//create job
Job job = Job.getInstance(configuration, this.getClass().getSimpleName()); job.setJarByClass(this.getClass());
//set job
//input:
Path inpath = new Path(args[0]); FileInputFormat.addInputPath(job, inpath);
//mapper
job.setMapperClass(WordCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class);

//reducer
job.setReducerClass(WordCountReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class);
//submit job
boolean isSucess = job.waitForCompletion(true);
return isSucess ? 0:1;
}
public static void main(String[] args) throws Exception { args = new String [] {
"hdfs://bigdata-senior01.beifeng.com:8020/user/beifeng/input/",

//
"hdfs://bigdata-senior01.beifeng.com:8020/user/beifeng/output9"
};
//run job
int status = new WordCountReducer().run(args);

System.exit(status);
}
}

對<key, value>在map和reduce階段的處理

package com.beifeng.bigdata.senior.hadoop.mapreduce;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; public class WordCountMapReduce {
//Step 1: Mapper Class
public static class WordCountMapper extends
Mapper<LongWritable, Text, Text, IntWritable>{
private Text mapOutputKey = new Text();
private final static IntWritable mapOutputValue = new IntWritable();
@Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
//line value
String lineValue = value.toString();
//split line words into separated word
String [] strs = lineValue.split(" ");
//record each value by "for" iterator to build <key, value>
for (String str : strs) {
}
}
}
//Step 2: Reducer Class
public static class WordCountReducer extends
Reducer<Text, IntWritable, Text, IntWritable>{
private IntWritable outputValue = new IntWritable(); @Override
protected void reduce(Text key, Iterable<IntWritable> values,
Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
int sum = 0;
//Iterator
for(IntWritable value : values){
sum += value.get();
}
//set output value
outputValue.set(sum);
//output
context.write(key, outputValue);
}
public int run(String[] args) {
// TODO Auto-generated method stub
return 0;
}
}
//Step 3: Driver
public int run(String[] args) throws Exception {
//read configuration file
Configuration configuration = new Configuration();
//create job
Job job = Job.getInstance(configuration, this.getClass().getSimpleName()); job.setJarByClass(this.getClass());
//set job
//input:
Path inpath = new Path(args[0]); FileInputFormat.addInputPath(job, inpath);
//mapper
job.setMapperClass(WordCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); //reducer
job.setReducerClass(WordCountReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class);
//submit job
boolean isSucess = job.waitForCompletion(true);
return isSucess ? 0:1;
}
public static void main(String[] args) throws Exception { args = new String [] {
"hdfs://bigdata-senior01.beifeng.com:8020/user/beifeng/input/",
"hdfs://bigdata-senior01.beifeng.com:8020/user/beifeng/output"
};
//run job
int status = new WordCountMapReduce().run(args); System.exit(status);
}
}

嘗試使用mapreduce來處理wordcount

package com.beifeng.bigdata.senior.hadoop.mapreduce;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class WordCountMapReduce {
//Step 1: Mapper Class
public static class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
private Text mapOutputKey = new Text();
private final static IntWritable mapOutputValue = new IntWritable(1);
@Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub System.out.println("map-in-0-key" + key.get() + " -- " + "map-in-value" + value.toString());
//line value
String lineValue = value.toString();
//split line words into separated word
String [] strs = lineValue.split(" ");
//record each value by "for" iterator to build <key, value> for (String str : strs) {
//set map output key mapOutputKey.set(str);
//output
context.write(mapOutputKey, mapOutputValue); }
}
}
//Step 2: Reducer Class
public static class WordCountReducer extends
Reducer<Text, IntWritable, Text, IntWritable>{
private IntWritable outputValue = new IntWritable(); @Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
// TODO Auto-generated method stub
int sum = 0;
//Iterator
for(IntWritable value : values){
sum += value.get();
}
//set output value
outputValue.set(sum);
//output
context.write(key, outputValue);
}
}
//Step 3: Driver
public int run(String[] args) throws Exception {
//read configuration file
Configuration configuration = new Configuration();
//create job
Job job = Job.getInstance(configuration, this.getClass().getSimpleName()); job.setJarByClass(this.getClass());
//set job
//input:
Path inpath = new Path(args[0]); FileInputFormat.addInputPath(job, inpath);
//output
Path outPath = new Path(args[1]); FileOutputFormat.setOutputPath(job, outPath); //mapper
job.setMapperClass(WordCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); //reducer
job.setReducerClass(WordCountReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class);
//submit job
boolean isSucess = job.waitForCompletion(true);
return isSucess ? 0 : 1;
}
public static void main(String[] args) throws Exception { args = new String [] {
"hdfs://bigdata-senior01.beifeng.com:8020/user/beifeng/input",
"hdfs://bigdata-senior01.beifeng.com:8020/user/beifeng/output13"};
//run job
int status = new WordCountMapReduce().run(args); System.exit(status);
}
}