前面我们介绍了：
向HBase中导入数据2：使用MapReduce从HDFS或本地文件中读取数据并写入HBase（只使用Map逐条查询）
为了提高插入效率，我们在前面只使用map的基础上增加使用reduce，思想是使用map-reduce操作，将rowkey相同的项规约到同一个reduce中，再在reduce中构建put对象实现批量插入
测试数据如下:
注意到有两条记录是相似的。
package cn.edu.shu.ces.chenjie.tianyi.hadoop;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
/***
* 使用MapReduce向HBase中导入数据
* @author chenjie
*
*/
public class HadoopConnectTest4
{
public static class HBaseHFileMapper extends Mapper<LongWritable, Text, Text, Text>
{
@Override
protected void map(LongWritable key, Text value, Context context)
{
String value_str = value.toString();
System.out.println("----map----" + value_str);
//将Text型数据转为字符串：
//1080x1920|Vivo X9|a8622b8a26ae679f0be82135f6529902|中国|湖南|益阳|wifi|2017-12-31 11:55:58
String values[] = value_str.split("\\|");
//按|隔开
if (values == null || values.length < 8)
return;
//过滤掉长度不符合要求的记录
String userID = values[2];
//取出用户:a8622b8a26ae679f0be82135f6529902
String time = values[7];
//取出时间:2017-12-31 11:55:58
String ymd = time.split(" ")[0];
//得到年月日：2017-12-31
String rowkey = userID + "-" + ymd;
//使用用户ID-年月日作为HBase表中的行键
try
{
context.write(new Text(rowkey), new Text(value_str));
}
catch (IOException e)
{
e.printStackTrace();
}
catch (InterruptedException e)
{
e.printStackTrace();
}
}
}
/**
* 注意这里的TableReducer
*/
public static class HBaseHFileReducer2 extends TableReducer<Text,Text,NullWritable>
{
@Override
protected void reduce(Text key, Iterable<Text> texts, Context context) throws IOException, InterruptedException
{
String rowkey = key.toString();
System.out.println("--reduce:" + rowkey);
Put p1 = new Put(Bytes.toBytes(rowkey));
//使用行键新建Put对象
for(Text value : texts)
{
String value_str = value.toString();
System.out.println("\t" + value_str);
String values[] = value_str.split("\\|");
//按|隔开
if (values == null || values.length < 8)
return;
//过滤掉长度不符合要求的记录
String time = values[7];
//取出时间:2017-12-31 11:55:58
String hms = time.split(" ")[1] + ":000";
//得到时分秒：11:55:58
//p1.addColumn(value.getFamily(), value.getQualifier(), value.getValue());
p1.addColumn("d".getBytes(), hms.getBytes(), value_str.getBytes());
//向put中增加一列，列族为d，列名为时分秒，值为原字符串
}
context.write(NullWritable.get(), p1);
}
}
private static final String HDFS = "hdfs://192.168.1.112:9000";// HDFS路径
private static final String INPATH = HDFS + "/tmp/clientdata10_3.txt";// 输入文件路径
public int run() throws IOException, ClassNotFoundException, InterruptedException
{
Configuration conf = HBaseConfiguration.create();
//任务的配置设置，configuration是一个任务的配置对象，封装了任务的配置信息
conf.set("hbase.zookeeper.quorum", "pc2:2181,pc3:2181,pc4:2181");
//设置zookeeper
conf.set("hbase.rootdir", "hdfs://pc2:9000/hbase");
//设置hbase根目录
conf.set("zookeeper.znode.parent", "/hbase");
Job job = Job.getInstance(conf, "HFile bulk load test");
// 生成一个新的任务对象并
job.setJarByClass(HadoopConnectTest4.class);
//设置driver类
job.setMapperClass(HBaseHFileMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
// 设置任务的map类和,map类输出结果是ImmutableBytesWritable和put类型
TableMapReduceUtil.initTableReducerJob("clientdata_test5", HBaseHFileReducer2.class, job);
// TableMapReduceUtil是HBase提供的工具类，会自动设置mapreuce提交到hbase任务的各种配置，封装了操作，只需要简单的设置即可
//设置表名为clientdata_test5，reducer类为空，job为此前设置号的job
job.setNumReduceTasks(1);
// 设置reduce过程，这里由map端的数据直接提交，不要使用reduce类，因而设置成null,并设置reduce的个数为0
FileInputFormat.addInputPath(job, new Path(INPATH));
// 设置输入文件路径
return (job.waitForCompletion(true) ? 0 : -1);
}
}运行结果：
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hadooplibs/slf4j-log4j12-1.7.7.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/slf4j-log4j12-1.7.5.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
2018-03-16 22:43:42,360 INFO [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1129)) - session.id is deprecated. Instead, use dfs.metrics.session-id
2018-03-16 22:43:42,366 INFO [main] jvm.JvmMetrics (JvmMetrics.java:init(76)) - Initializing JVM Metrics with processName=JobTracker, sessionId=
2018-03-16 22:43:42,437 INFO [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1129)) - io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
2018-03-16 22:43:42,717 WARN [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(64)) - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
2018-03-16 22:43:42,729 WARN [main] mapreduce.JobResourceUploader (JobResourceUploader.java:uploadFiles(171)) - No job jar file set. User classes may not be found. See Job or Job#setJar(String).
2018-03-16 22:43:44,094 INFO [main] input.FileInputFormat (FileInputFormat.java:listStatus(281)) - Total input paths to process : 1
2018-03-16 22:43:44,181 INFO [main] mapreduce.JobSubmitter (JobSubmitter.java:submitJobInternal(199)) - number of splits:1
2018-03-16 22:43:44,192 INFO [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1129)) - io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
2018-03-16 22:43:44,355 INFO [main] mapreduce.JobSubmitter (JobSubmitter.java:printTokens(288)) - Submitting tokens for job: job_local597129759_0001
2018-03-16 22:43:46,410 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/zookeeper-3.4.6.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424485/zookeeper-3.4.6.jar
2018-03-16 22:43:46,411 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hadooplibs/hadoop-mapreduce-client-core-2.6.5.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424486/hadoop-mapreduce-client-core-2.6.5.jar
2018-03-16 22:43:46,412 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/hbase-client-1.2.6.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424487/hbase-client-1.2.6.jar
2018-03-16 22:43:46,413 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hadooplibs/protobuf-java-2.5.0.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424488/protobuf-java-2.5.0.jar
2018-03-16 22:43:46,416 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/htrace-core-3.1.0-incubating.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424489/htrace-core-3.1.0-incubating.jar
2018-03-16 22:43:46,518 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/metrics-core-2.2.0.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424490/metrics-core-2.2.0.jar
2018-03-16 22:43:46,687 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hadooplibs/hadoop-common-2.6.5.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424491/hadoop-common-2.6.5.jar
2018-03-16 22:43:46,691 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/hbase-common-1.2.6.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424492/hbase-common-1.2.6.jar
2018-03-16 22:43:46,692 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/hbase-hadoop-compat-1.2.6.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424493/hbase-hadoop-compat-1.2.6.jar
2018-03-16 22:43:46,694 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/netty-all-4.0.23.Final.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424494/netty-all-4.0.23.Final.jar
2018-03-16 22:43:46,696 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/hbase-prefix-tree-1.2.6.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424495/hbase-prefix-tree-1.2.6.jar
2018-03-16 22:43:46,698 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/hbase-protocol-1.2.6.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424496/hbase-protocol-1.2.6.jar
2018-03-16 22:43:46,707 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hbaselibs/hbase-server-1.2.6.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424497/hbase-server-1.2.6.jar
2018-03-16 22:43:46,708 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:setup(165)) - Localized file:/D:/SourceCode/MyEclipse2014Workspace/HDFS2HBaseByHadoopMR/hadooplibs/guava-11.0.2.jar as file:/tmp/hadoop-Administrator/mapred/local/1521211424498/guava-11.0.2.jar
2018-03-16 22:43:46,795 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424485/zookeeper-3.4.6.jar
2018-03-16 22:43:46,796 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424486/hadoop-mapreduce-client-core-2.6.5.jar
2018-03-16 22:43:46,796 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424487/hbase-client-1.2.6.jar
2018-03-16 22:43:46,796 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424488/protobuf-java-2.5.0.jar
2018-03-16 22:43:46,796 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424489/htrace-core-3.1.0-incubating.jar
2018-03-16 22:43:46,796 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424490/metrics-core-2.2.0.jar
2018-03-16 22:43:46,797 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424491/hadoop-common-2.6.5.jar
2018-03-16 22:43:46,797 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424492/hbase-common-1.2.6.jar
2018-03-16 22:43:46,797 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424493/hbase-hadoop-compat-1.2.6.jar
2018-03-16 22:43:46,799 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424494/netty-all-4.0.23.Final.jar
2018-03-16 22:43:46,799 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424495/hbase-prefix-tree-1.2.6.jar
2018-03-16 22:43:46,799 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424496/hbase-protocol-1.2.6.jar
2018-03-16 22:43:46,799 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424497/hbase-server-1.2.6.jar
2018-03-16 22:43:46,800 INFO [main] mapred.LocalDistributedCacheManager (LocalDistributedCacheManager.java:makeClassLoader(234)) - file:/D:/tmp/hadoop-Administrator/mapred/local/1521211424498/guava-11.0.2.jar
2018-03-16 22:43:46,807 INFO [main] mapreduce.Job (Job.java:submit(1301)) - The url to track the job: http://localhost:8080/
2018-03-16 22:43:46,808 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1346)) - Running job: job_local597129759_0001
2018-03-16 22:43:46,818 INFO [Thread-24] mapred.LocalJobRunner (LocalJobRunner.java:createOutputCommitter(471)) - OutputCommitter set in config null
2018-03-16 22:43:46,896 INFO [Thread-24] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1129)) - io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
2018-03-16 22:43:46,898 INFO [Thread-24] mapred.LocalJobRunner (LocalJobRunner.java:createOutputCommitter(489)) - OutputCommitter is org.apache.hadoop.hbase.mapreduce.TableOutputCommitter
2018-03-16 22:43:46,984 INFO [Thread-24] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(448)) - Waiting for map tasks
2018-03-16 22:43:46,987 INFO [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:run(224)) - Starting task: attempt_local597129759_0001_m_000000_0
2018-03-16 22:43:47,079 INFO [LocalJobRunner Map Task Executor #0] util.ProcfsBasedProcessTree (ProcfsBasedProcessTree.java:isAvailable(181)) - ProcfsBasedProcessTree currently is supported only on Linux.
2018-03-16 22:43:47,172 INFO [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:initialize(587)) - Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@76e774ab
2018-03-16 22:43:47,181 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:runNewMapper(753)) - Processing split: hdfs://192.168.1.112:9000/tmp/clientdata10_3.txt:0+1130
2018-03-16 22:43:47,237 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:setEquator(1202)) - (EQUATOR) 0 kvi 26214396(104857584)
2018-03-16 22:43:47,237 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(995)) - mapreduce.task.io.sort.mb: 100
2018-03-16 22:43:47,237 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(996)) - soft limit at 83886080
2018-03-16 22:43:47,238 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(997)) - bufstart = 0; bufvoid = 104857600
2018-03-16 22:43:47,238 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:init(998)) - kvstart = 26214396; length = 6553600
2018-03-16 22:43:47,241 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:createSortingCollector(402)) - Map output collector class = org.apache.hadoop.mapred.MapTask$MapOutputBuffer
2018-03-16 22:43:47,811 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) - Job job_local597129759_0001 running in uber mode : false
2018-03-16 22:43:47,813 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1374)) - map 0% reduce 0%
2018-03-16 22:43:48,365 INFO [LocalJobRunner Map Task Executor #0] input.LineRecordReader (LineRecordReader.java:skipUtfByteOrderMark(157)) - Found UTF-8 BOM and skipped it
----map----1080x1920|Vivo X9|a8622b8a26ae679f0be82135f6529902|中国|湖南|益阳|wifi|2017-12-31 11:55:58
----map----1080x1920|Vivo X9|1c4e5de968280d746667df16ea851fbc|中国|安徽|池州|wifi|2017-12-31 10:15:14
----map----1080x1812|HUAWEI NEM-AL10|1c7fbee71125ff575dc8f8eee233e359|中国|贵州|贵阳|wifi|2017-12-31 10:14:22
----map----720x1280|Vivo Y55A|ecaa9c2588bb551e666861b1bf9352fd|中国|江西|unknown|wifi|2017-12-31 20:13:40
----map----1080x1776|HUAWEI KIW-CL00|a70132d9e71afe2842cb140c70cf8b92|中国|江苏|南京|ctnet|2017-12-31 09:36:38
----map----720x1280|Meizu M578CA|4f77fe8ebb9cdc08c5757e17de3d3dc0|中国|江西|南昌|ctnet|2017-12-31 15:24:41
----map----1920x1080|OPPO R9sk|934263fff13ab52b523dcae794f2c09b|中国|辽宁|unknown|ctnet|2017-12-31 06:41:48
----map----1184x720|HUAWEI JMM-AL10|511dccb14d1d28420940a4d7c1d781f0|中国|山东|威海|wifi|2017-12-31 14:37:33
----map----1080x1920|Vivo V3Max A|c52869a5a84136aeaf8cd452fe7f0d44|中国|江苏|南京|wifi|2017-12-31 09:03:44
----map----1920x1080|OPPO A77|5a13b7c9cf5fd5057bd9af9506caef2e|中国|浙江|unknown|ctnet|2017-12-31 12:06:14
----map----1080x1812|HUAWEI NEM-AL10|1c7fbee71125ff575dc8f8eee233e359|中国|贵州|贵阳|wifi|2017-12-31 00:00:00
2018-03-16 22:43:48,372 INFO [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) -
2018-03-16 22:43:48,375 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1457)) - Starting flush of map output
2018-03-16 22:43:48,375 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1475)) - Spilling map output
2018-03-16 22:43:48,375 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1476)) - bufstart = 0; bufend = 1602; bufvoid = 104857600
2018-03-16 22:43:48,375 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:flush(1478)) - kvstart = 26214396(104857584); kvend = 26214356(104857424); length = 41/6553600
2018-03-16 22:43:48,391 INFO [LocalJobRunner Map Task Executor #0] mapred.MapTask (MapTask.java:sortAndSpill(1660)) - Finished spill 0
2018-03-16 22:43:48,402 INFO [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:done(1001)) - Task:attempt_local597129759_0001_m_000000_0 is done. And is in the process of committing
2018-03-16 22:43:48,413 INFO [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - map
2018-03-16 22:43:48,413 INFO [LocalJobRunner Map Task Executor #0] mapred.Task (Task.java:sendDone(1121)) - Task 'attempt_local597129759_0001_m_000000_0' done.
2018-03-16 22:43:48,413 INFO [LocalJobRunner Map Task Executor #0] mapred.LocalJobRunner (LocalJobRunner.java:run(249)) - Finishing task: attempt_local597129759_0001_m_000000_0
2018-03-16 22:43:48,413 INFO [Thread-24] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(456)) - map task executor complete.
2018-03-16 22:43:48,416 INFO [Thread-24] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(448)) - Waiting for reduce tasks
2018-03-16 22:43:48,416 INFO [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(302)) - Starting task: attempt_local597129759_0001_r_000000_0
2018-03-16 22:43:48,443 INFO [pool-7-thread-1] util.ProcfsBasedProcessTree (ProcfsBasedProcessTree.java:isAvailable(181)) - ProcfsBasedProcessTree currently is supported only on Linux.
2018-03-16 22:43:48,645 INFO [pool-7-thread-1] mapred.Task (Task.java:initialize(587)) - Using ResourceCalculatorProcessTree : org.apache.hadoop.yarn.util.WindowsBasedProcessTree@2620e87b
2018-03-16 22:43:48,648 INFO [pool-7-thread-1] mapred.ReduceTask (ReduceTask.java:run(362)) - Using ShuffleConsumerPlugin: org.apache.hadoop.mapreduce.task.reduce.Shuffle@251e41ef
2018-03-16 22:43:48,665 INFO [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:<init>(197)) - MergerManager: memoryLimit=1316801664, maxSingleShuffleLimit=329200416, mergeThreshold=869089152, ioSortFactor=10, memToMemMergeOutputsThreshold=10
2018-03-16 22:43:48,668 INFO [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(61)) - attempt_local597129759_0001_r_000000_0 Thread started: EventFetcher for fetching Map Completion Events
2018-03-16 22:43:48,704 INFO [localfetcher#1] reduce.LocalFetcher (LocalFetcher.java:copyMapOutput(141)) - localfetcher#1 about to shuffle output of map attempt_local597129759_0001_m_000000_0 decomp: 1626 len: 1630 to MEMORY
2018-03-16 22:43:48,714 INFO [localfetcher#1] reduce.InMemoryMapOutput (InMemoryMapOutput.java:shuffle(100)) - Read 1626 bytes from map-output for attempt_local597129759_0001_m_000000_0
2018-03-16 22:43:48,717 INFO [localfetcher#1] reduce.MergeManagerImpl (MergeManagerImpl.java:closeInMemoryFile(315)) - closeInMemoryFile -> map-output of size: 1626, inMemoryMapOutputs.size() -> 1, commitMemory -> 0, usedMemory ->1626
2018-03-16 22:43:48,720 INFO [EventFetcher for fetching Map Completion Events] reduce.EventFetcher (EventFetcher.java:run(76)) - EventFetcher is interrupted.. Returning
2018-03-16 22:43:48,721 INFO [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 1 / 1 copied.
2018-03-16 22:43:48,721 INFO [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(687)) - finalMerge called with 1 in-memory map-outputs and 0 on-disk map-outputs
2018-03-16 22:43:48,739 INFO [pool-7-thread-1] mapred.Merger (Merger.java:merge(597)) - Merging 1 sorted segments
2018-03-16 22:43:48,740 INFO [pool-7-thread-1] mapred.Merger (Merger.java:merge(696)) - Down to the last merge-pass, with 1 segments left of total size: 1580 bytes
2018-03-16 22:43:48,746 INFO [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(754)) - Merged 1 segments, 1626 bytes to disk to satisfy reduce memory limit
2018-03-16 22:43:48,747 INFO [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(784)) - Merging 1 files, 1630 bytes from disk
2018-03-16 22:43:48,748 INFO [pool-7-thread-1] reduce.MergeManagerImpl (MergeManagerImpl.java:finalMerge(799)) - Merging 0 segments, 0 bytes from memory into reduce
2018-03-16 22:43:48,748 INFO [pool-7-thread-1] mapred.Merger (Merger.java:merge(597)) - Merging 1 sorted segments
2018-03-16 22:43:48,752 INFO [pool-7-thread-1] mapred.Merger (Merger.java:merge(696)) - Down to the last merge-pass, with 1 segments left of total size: 1580 bytes
2018-03-16 22:43:48,756 INFO [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - 1 / 1 copied.
2018-03-16 22:43:48,816 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1374)) - map 100% reduce 0%
2018-03-16 22:43:49,922 INFO [pool-7-thread-1] zookeeper.RecoverableZooKeeper (RecoverableZooKeeper.java:<init>(120)) - Process identifier=hconnection-0x478f1f48 connecting to ZooKeeper ensemble=pc2:2181,pc3:2181,pc4:2181
2018-03-16 22:43:49,930 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT
2018-03-16 22:43:49,931 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:host.name=chen
2018-03-16 22:43:49,931 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.version=1.8.0_74
2018-03-16 22:43:49,931 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.vendor=Oracle Corporation
2018-03-16 22:43:49,931 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.home=D:\Java\jdk1.8.0_74\jre
2018-03-16 22:43:49,931 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.class.path=D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\bin;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\activation-1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\aopalliance-1.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\apacheds-i18n-2.0.0-M15.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\apacheds-kerberos-codec-2.0.0-M15.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\api-asn1-api-1.0.0-M20.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\api-util-1.0.0-M20.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\asm-3.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\aspectjweaver-1.6.11.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\avro-1.7.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-beanutils-core-1.8.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-cli-1.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-codec-1.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-collections-3.2.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-compress-1.4.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-configuration-1.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-dbcp-1.2.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-digester-1.8.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-el-1.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-fileupload-1.2.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-httpclient-3.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-io-2.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-lang-2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-logging-1.1.3.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-math3-3.1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-net-3.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\commons-pool-1.5.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\curator-client-2.6.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\curator-framework-2.6.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\curator-recipes-2.6.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\fastjson-1.1.41.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\gson-2.3.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\guava-11.0.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-annotations-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-auth-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-common-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-hdfs-2.6.5-tests.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-hdfs-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-hdfs-nfs-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-app-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-common-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-core-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-hs-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-hs-plugins-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-jobclient-2.6.5-tests.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-jobclient-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-client-shuffle-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-mapreduce-examples-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-nfs-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-api-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-applications-distributedshell-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-applications-unmanaged-am-launcher-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-client-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-common-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-registry-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-server-applicationhistoryservice-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-server-common-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-server-nodemanager-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-server-resourcemanager-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-server-tests-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hadoop-yarn-server-web-proxy-2.6.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\hamcrest-core-1.3.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\htrace-core-3.0.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\httpclient-4.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\httpcore-4.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jackson-core-asl-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jackson-jaxrs-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jackson-mapper-asl-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jackson-xc-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\java-xmlbuilder-0.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\javax.mail-1.5.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jaxb-api-2.2.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jaxb-impl-2.2.3-1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jedis-2.1.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jersey-core-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jersey-json-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jersey-server-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jets3t-0.9.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jettison-1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jetty-6.1.26.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jetty-util-6.1.26.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jsch-0.1.42.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\jsr305-1.3.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\junit-4.11.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\libfb303-0.9.3.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\log4j-1.2.17.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\mockito-all-1.8.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\netty-3.6.2.Final.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\paranamer-2.3.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\protobuf-java-2.5.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\protostuff-api-1.1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\protostuff-collectionschema-1.1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\protostuff-core-1.1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\protostuff-runtime-1.1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\servlet-api-2.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\slf4j-api-1.7.7.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\slf4j-log4j12-1.7.7.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\snappy-java-1.0.4.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\stax-api-1.0-2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\xmlenc-0.52.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hadooplibs\xz-1.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\activation-1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\aopalliance-1.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\apacheds-i18n-2.0.0-M15.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\apacheds-kerberos-codec-2.0.0-M15.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\api-asn1-api-1.0.0-M20.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\api-util-1.0.0-M20.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\asm-3.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\avro-1.7.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-beanutils-1.7.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-beanutils-core-1.8.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-cli-1.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-codec-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-collections-3.2.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-compress-1.4.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-configuration-1.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-daemon-1.0.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-digester-1.8.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-el-1.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-httpclient-3.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-io-2.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-lang-2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-logging-1.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-math-2.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-math3-3.1.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\commons-net-3.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\disruptor-3.3.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\findbugs-annotations-1.3.9-1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\guava-12.0.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\guice-3.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\guice-servlet-3.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-annotations-1.2.6-tests.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-annotations-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-client-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-common-1.2.6-tests.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-common-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-examples-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-external-blockcache-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-hadoop-compat-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-hadoop2-compat-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-it-1.2.6-tests.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-it-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-prefix-tree-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-procedure-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-protocol-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-resource-bundle-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-rest-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-server-1.2.6-tests.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-server-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-shell-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\hbase-thrift-1.2.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\htrace-core-3.1.0-incubating.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\httpclient-4.2.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\httpcore-4.4.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jackson-core-asl-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jackson-jaxrs-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jackson-mapper-asl-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jackson-xc-1.9.13.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jamon-runtime-2.4.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jasper-compiler-5.5.23.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jasper-runtime-5.5.23.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\java-xmlbuilder-0.4.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\javax.inject-1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jaxb-api-2.2.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jaxb-impl-2.2.3-1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jcodings-1.0.8.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jersey-client-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jersey-core-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jersey-guice-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jersey-json-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jersey-server-1.9.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jets3t-0.9.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jettison-1.3.3.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jetty-6.1.26.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jetty-sslengine-6.1.26.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jetty-util-6.1.26.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\joni-2.1.2.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jruby-complete-1.6.8.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jsch-0.1.42.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jsp-2.1-6.1.14.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\jsp-api-2.1-6.1.14.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\junit-4.12.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\leveldbjni-all-1.8.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\libthrift-0.9.3.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\log4j-1.2.17.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\metrics-core-2.2.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\netty-all-4.0.23.Final.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\paranamer-2.3.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\protobuf-java-2.5.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\slf4j-api-1.7.7.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\slf4j-log4j12-1.7.5.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\snappy-java-1.0.4.1.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\spymemcached-2.11.6.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\xmlenc-0.52.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\xz-1.0.jar;D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR\hbaselibs\zookeeper-3.4.6.jar
2018-03-16 22:43:49,932 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.library.path=D:\Java\jdk1.8.0_74\bin;C:\Windows\Sun\Java\bin;C:\Windows\system32;C:\Windows;C:\Program Files (x86)\ARM\ADSv1_2\bin;D:\Python36-32\Scripts\;D:\Python36-32\;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Intel\OpenCL SDK\2.0\bin\x86;C:\Program Files (x86)\Intel\OpenCL SDK\2.0\bin\x64;C:\Program Files (x86)\NVIDIA Corporation\PhysX\Common;D:\Java\jdk1.8.0_74\bin;D:\Java\jdk1.8.0_74\jre\bin;D:\mysql-5.6.24-winx64\bin;C:\Program Files (x86)\GtkSharp\2.12\bin;C:\Program Files\OpenVPN\bin;C:\Program Files\Git\cmd;D:\scala-2.12.4\bin;D:\jython2.7.0;D:\jython2.7.0\bin;D:\Program Files\nodejs\;D:\apache-maven-3.5.0\bin;D:\hadoop-2.6.5\bin;D:\spark-1.6.3-bin-hadoop2.6\spark-1.6.3-bin-hadoop2.6\bin;C:\strawberry\c\bin;C:\strawberry\perl\bin;C:\Program Files\TortoiseSVN\bin;D:\ldxfiles\apache-ant-1.10.1-bin\apache-ant-1.10.1\bin;C:\Program Files (x86)\Microsoft Visual Studio\Common\Tools\WinNT;C:\Program Files (x86)\Microsoft Visual Studio\Common\MSDev98\Bin;C:\Program Files (x86)\Microsoft Visual Studio\Common\Tools;C:\Program Files (x86)\Microsoft Visual Studio\VC98\bin;C:\Users\Administrator\AppData\Roaming\npm;C:\Users\Administrator\AppData\Local\Programs\Fiddler;.
2018-03-16 22:43:49,932 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.io.tmpdir=C:\Users\ADMINI~1\AppData\Local\Temp\
2018-03-16 22:43:49,932 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:java.compiler=<NA>
2018-03-16 22:43:49,932 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:os.name=Windows 7
2018-03-16 22:43:49,932 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:os.arch=amd64
2018-03-16 22:43:49,933 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:os.version=6.1
2018-03-16 22:43:49,933 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:user.name=Administrator
2018-03-16 22:43:49,933 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:user.home=C:\Users\Administrator
2018-03-16 22:43:49,933 INFO [pool-7-thread-1] zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:user.dir=D:\SourceCode\MyEclipse2014Workspace\HDFS2HBaseByHadoopMR
2018-03-16 22:43:49,934 INFO [pool-7-thread-1] zookeeper.ZooKeeper (ZooKeeper.java:<init>(438)) - Initiating client connection, connectString=pc2:2181,pc3:2181,pc4:2181 sessionTimeout=90000 watcher=hconnection-0x478f1f480x0, quorum=pc2:2181,pc3:2181,pc4:2181, baseZNode=/hbase
2018-03-16 22:43:49,960 INFO [pool-7-thread-1-SendThread(pc3:2181)] zookeeper.ClientCnxn (ClientCnxn.java:logStartConnect(975)) - Opening socket connection to server pc3/192.168.1.81:2181. Will not attempt to authenticate using SASL (unknown error)
2018-03-16 22:43:49,966 INFO [pool-7-thread-1-SendThread(pc3:2181)] zookeeper.ClientCnxn (ClientCnxn.java:primeConnection(852)) - Socket connection established to pc3/192.168.1.81:2181, initiating session
2018-03-16 22:43:49,984 INFO [pool-7-thread-1-SendThread(pc3:2181)] zookeeper.ClientCnxn (ClientCnxn.java:onConnected(1235)) - Session establishment complete on server pc3/192.168.1.81:2181, sessionid = 0x200001082220017, negotiated timeout = 40000
2018-03-16 22:43:50,110 INFO [pool-7-thread-1] mapreduce.TableOutputFormat (TableOutputFormat.java:<init>(108)) - Created table instance for clientdata_test5
2018-03-16 22:43:50,110 INFO [pool-7-thread-1] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1129)) - mapred.skip.on is deprecated. Instead, use mapreduce.job.skiprecords
--reduce:1c4e5de968280d746667df16ea851fbc-2017-12-31
1080x1920|Vivo X9|1c4e5de968280d746667df16ea851fbc|中国|安徽|池州|wifi|2017-12-31 10:15:14
--reduce:1c7fbee71125ff575dc8f8eee233e359-2017-12-31
1080x1812|HUAWEI NEM-AL10|1c7fbee71125ff575dc8f8eee233e359|中国|贵州|贵阳|wifi|2017-12-31 00:00:00
1080x1812|HUAWEI NEM-AL10|1c7fbee71125ff575dc8f8eee233e359|中国|贵州|贵阳|wifi|2017-12-31 10:14:22
--reduce:4f77fe8ebb9cdc08c5757e17de3d3dc0-2017-12-31
720x1280|Meizu M578CA|4f77fe8ebb9cdc08c5757e17de3d3dc0|中国|江西|南昌|ctnet|2017-12-31 15:24:41
--reduce:511dccb14d1d28420940a4d7c1d781f0-2017-12-31
1184x720|HUAWEI JMM-AL10|511dccb14d1d28420940a4d7c1d781f0|中国|山东|威海|wifi|2017-12-31 14:37:33
--reduce:5a13b7c9cf5fd5057bd9af9506caef2e-2017-12-31
1920x1080|OPPO A77|5a13b7c9cf5fd5057bd9af9506caef2e|中国|浙江|unknown|ctnet|2017-12-31 12:06:14
--reduce:934263fff13ab52b523dcae794f2c09b-2017-12-31
1920x1080|OPPO R9sk|934263fff13ab52b523dcae794f2c09b|中国|辽宁|unknown|ctnet|2017-12-31 06:41:48
--reduce:a70132d9e71afe2842cb140c70cf8b92-2017-12-31
1080x1776|HUAWEI KIW-CL00|a70132d9e71afe2842cb140c70cf8b92|中国|江苏|南京|ctnet|2017-12-31 09:36:38
--reduce:a8622b8a26ae679f0be82135f6529902-2017-12-31
1080x1920|Vivo X9|a8622b8a26ae679f0be82135f6529902|中国|湖南|益阳|wifi|2017-12-31 11:55:58
--reduce:c52869a5a84136aeaf8cd452fe7f0d44-2017-12-31
1080x1920|Vivo V3Max A|c52869a5a84136aeaf8cd452fe7f0d44|中国|江苏|南京|wifi|2017-12-31 09:03:44
--reduce:ecaa9c2588bb551e666861b1bf9352fd-2017-12-31
720x1280|Vivo Y55A|ecaa9c2588bb551e666861b1bf9352fd|中国|江西|unknown|wifi|2017-12-31 20:13:40
2018-03-16 22:43:50,637 INFO [pool-7-thread-1] client.ConnectionManager$HConnectionImplementation (ConnectionManager.java:closeZooKeeperWatcher(1710)) - Closing zookeeper sessionid=0x200001082220017
2018-03-16 22:43:50,646 INFO [pool-7-thread-1] zookeeper.ZooKeeper (ZooKeeper.java:close(684)) - Session: 0x200001082220017 closed
2018-03-16 22:43:50,646 INFO [pool-7-thread-1-EventThread] zookeeper.ClientCnxn (ClientCnxn.java:run(512)) - EventThread shut down
2018-03-16 22:43:50,656 INFO [pool-7-thread-1] mapred.Task (Task.java:done(1001)) - Task:attempt_local597129759_0001_r_000000_0 is done. And is in the process of committing
2018-03-16 22:43:50,658 INFO [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:statusUpdate(591)) - reduce > reduce
2018-03-16 22:43:50,658 INFO [pool-7-thread-1] mapred.Task (Task.java:sendDone(1121)) - Task 'attempt_local597129759_0001_r_000000_0' done.
2018-03-16 22:43:50,658 INFO [pool-7-thread-1] mapred.LocalJobRunner (LocalJobRunner.java:run(325)) - Finishing task: attempt_local597129759_0001_r_000000_0
2018-03-16 22:43:50,660 INFO [Thread-24] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(456)) - reduce task executor complete.
2018-03-16 22:43:50,816 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1374)) - map 100% reduce 100%
2018-03-16 22:43:50,817 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Job job_local597129759_0001 completed successfully
2018-03-16 22:43:50,861 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1392)) - Counters: 38
File System Counters
FILE: Number of bytes read=43542990
FILE: Number of bytes written=44562580
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
HDFS: Number of bytes read=2260
HDFS: Number of bytes written=0
HDFS: Number of read operations=6
HDFS: Number of large read operations=0
HDFS: Number of write operations=0
Map-Reduce Framework
Map input records=11
Map output records=11
Map output bytes=1602
Map output materialized bytes=1630
Input split bytes=113
Combine input records=0
Combine output records=0
Reduce input groups=10
Reduce shuffle bytes=1630
Reduce input records=11
Reduce output records=10
Spilled Records=22
Shuffled Maps =1
Failed Shuffles=0
Merged Map outputs=1
GC time elapsed (ms)=18
CPU time spent (ms)=0
Physical memory (bytes) snapshot=0
Virtual memory (bytes) snapshot=0
Total committed heap usage (bytes)=457703424
Shuffle Errors
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
File Input Format Counters
Bytes Read=1130
File Output Format Counters
Bytes Written=0
注意到rowkey相同的行规约到了一起
大量数据测试：
扫一扫
7433
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
