haddop 任务 start.sh

hadoop_bin="/usr/bin/hadoop"
hadoop_streaming_file="/opt/cloudera/parcels/CDH-5.11.0-1.cdh5.11.0.p0.34/lib/hadoop-mapreduce/hadoop-streaming.jar"
hadoop_input_dir="hdfs://nameservice1/user/cid_pv_dir_combine"
hadoop_output_dir="hdfs://nameservice1/user/cid_most_simiar_combine"
${hadoop_bin} fs -rm -r ${hadoop_output_dir}
${hadoop_bin} jar ${hadoop_streaming_file} 

-archives test.tar
-file mapper_cid_most_similar.py
-file reducer_cid_most_similar.py
-file uid_cids.model
-file uid_cids.bin
-file uid_cids.model.docvecs.doctag_syn0.npy
-file uid_cids.model.wv.syn0.npy
-file uid_cids.model.syn1neg.npy
-inputformat TextInputFormat
-input $hadoop_input_dir
-output $hadoop_output_dir
-mapper “test.tar/test/bin/python mapper_cid_most_similar.py”
-reducer “test.tar/test/bin/python reducer_cid_most_similar.py”
-jobconf mapred.job.map.capacity=100
-jobconf mapred.job.reduce.capacity=100
-jobconf mapreduce.map.memory.mb=38192
-jobconf mapreduce.reduce.memory.mb=38192
-jobconf mapred.map.tasks=100
-jobconf mapred.reduce.tasks=0
-jobconf mapred.map.maxrunning=200
-jobconf mapred.reduce.maxrunning=200
-jobconf mapred.job.name=“doc2vec generate uid_rec_cids”
-jobconf mapred.reduce.slowstart.completed.maps=0.99
-jobconf stream.num.map.output.key.fields=2
-jobconf num.key.fields.for.partition=1
-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner
-jobconf mapred.job.priority=NORMAL

${hadoop_bin} fs -text ${hadoop_output_dir}/* >cid_most_simiar