Hadoop 实战之分析专利引用数据集（一）

环境：Vmware 8.0 和Ubuntu11.04

Hadoop 实战之分析专利引用数据集（一）---计算专利引用数据并排序

第一步：首先创建一个工程命名为HadoopTest.目录结构如下图:

第二步: 在/home/tanglg1987目录下新建一个start.sh脚本文件，每次启动虚拟机都要删除/tmp目录下的全部文件，重新格式化namenode，代码如下：

sudo rm -rf /tmp/*
rm -rf /home/tanglg1987/hadoop-0.20.2/logs
hadoop namenode -format
hadoop datanode -format
start-all.sh
hadoop fs -mkdir input 
hadoop dfsadmin -safemode leave

第三步：给start.sh增加执行权限并启动hadoop伪分布式集群，代码如下:

chmod 777 /home/tanglg1987/ start.sh
./start.sh 

执行过程如下：

12/10/15 23:05:38 INFO namenode.NameNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG: host = tanglg1987/127.0.1.1
STARTUP_MSG: args = [-format]
STARTUP_MSG: version = 0.20.2
STARTUP_MSG: build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20 -r 911707; compiled by 'chrisdo' on Fri Feb 19 08:07:34 UTC 2010
************************************************************/
12/10/15 23:05:39 INFO namenode.FSNamesystem: fsOwner=tanglg1987,tanglg1987,adm,dialout,cdrom,plugdev,lpadmin,admin,sambashare
12/10/15 23:05:39 INFO namenode.FSNamesystem: supergroup=supergroup
12/10/15 23:05:39 INFO namenode.FSNamesystem: isPermissionEnabled=true
12/10/15 23:05:39 INFO common.Storage: Image file of size 100 saved in 0 seconds.
12/10/15 23:05:39 INFO common.Storage: Storage directory /tmp/hadoop-tanglg1987/dfs/name has been successfully formatted.
12/10/15 23:05:39 INFO namenode.NameNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at tanglg1987/127.0.1.1
************************************************************/
12/10/15 23:05:40 INFO datanode.DataNode: STARTUP_MSG:
/************************************************************
STARTUP_MSG: Starting DataNode
STARTUP_MSG: host = tanglg1987/127.0.1.1
STARTUP_MSG: args = [-format]
STARTUP_MSG: version = 0.20.2
STARTUP_MSG: build = https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20 -r 911707; compiled by 'chrisdo' on Fri Feb 19 08:07:34 UTC 2010
************************************************************/
Usage: java DataNode
[-rollback]
12/10/15 23:05:40 INFO datanode.DataNode: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down DataNode at tanglg1987/127.0.1.1
************************************************************/
starting namenode, logging to /home/tanglg1987/hadoop-0.20.2/bin/../logs/hadoop-tanglg1987-namenode-tanglg1987.out
localhost: starting datanode, logging to /home/tanglg1987/hadoop-0.20.2/bin/../logs/hadoop-tanglg1987-datanode-tanglg1987.out
localhost: starting secondarynamenode, logging to /home/tanglg1987/hadoop-0.20.2/bin/../logs/hadoop-tanglg1987-secondarynamenode-tanglg1987.out
starting jobtracker, logging to /home/tanglg1987/hadoop-0.20.2/bin/../logs/hadoop-tanglg1987-jobtracker-tanglg1987.out
localhost: starting tasktracker, logging to /home/tanglg1987/hadoop-0.20.2/bin/../logs/hadoop-tanglg1987-tasktracker-tanglg1987.out
Safe mode is OFF

第四步：上传本地文件到hdfs

在专利局http://data.nber.org/patents/网站下载专利数据

http://data.nber.org/patents/apat63_99.zip

hadoop fs -put /home/tanglg1987/cite75_99.txt input

五步：新建一个MyJob.java，代码如下：

package com.baison.action;
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public class MyJob extends Configured implements Tool {   
    public static class MapClass extends MapReduceBase
        implements Mapper<Text, Text, Text, Text> {        
        public void map(Text key, Text value,
                        OutputCollector<Text, Text> output,
                        Reporter reporter) throws IOException {                        
            output.collect(value, key);
        }
    }    
    public static class Reduce extends MapReduceBase
        implements Reducer<Text, Text, Text, Text> {
        public void reduce(Text key, Iterator<Text> values,
                           OutputCollector<Text, Text> output,
                           Reporter reporter) throws IOException {                           
            String csv = "";
            while (values.hasNext()) {
                if (csv.length() > 0) csv += ",";
                csv += values.next().toString();
            }
            output.collect(key, new Text(csv));
        }
    }    
    public int run(String[] args) throws Exception {
     for (String string : args) {
   System.out.println(string);
  }
        Configuration conf = getConf();        
        JobConf job = new JobConf(conf, MyJob.class);        
        Path in = new Path(args[0]);
        Path out = new Path(args[1]);
        FileInputFormat.setInputPaths(job, in);
        FileOutputFormat.setOutputPath(job, out);        
        job.setJobName("MyJob");
        job.setMapperClass(MapClass.class);
        job.setReducerClass(Reduce.class);        
        job.setInputFormat(KeyValueTextInputFormat.class);
        job.setOutputFormat(TextOutputFormat.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        job.set("key.value.separator.in.input.line", ",");       
        JobClient.runJob(job);       
        return 0;
    }
    public static void main(String[] args) throws Exception { 
     String [] arg={"hdfs://localhost:9100/user/tanglg1987/input/cite75_99.txt","hdfs://localhost:9100/user/tanglg1987/output"};
        int res = ToolRunner.run(new Configuration(), new MyJob(), arg);       
        System.exit(res);
    }
}