package my.test;
import java.util.Map.Entry;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.*;
// ConfigPrinter:print all settings and properties of hadoop
public class ConfigPrinter extends Configured implements Tool {
/*
static {
Configuration.addDefaultResource("hdfs-default.xml");
Configuration.addDefaultResource("hdfs-site.xml");
Configuration.addDefaultResource("mapred-default.xml");
Configuration.addDefaultResource("mapred-site.xml");
//Configuration.addDefaultResource("test.xml");
}
*/
@Override
public int run(String[] args) throws Exception {
Configuration conf = getConf();
conf.addResource("test2.xml");
//设置job才会加载mapred-site.xml和hdfs-site.xml
Job job = new Job(conf, "print args");
job.setJarByClass(ConfigPrinter.class);
//System.setProperty("test1", "mytest");
for (Entry<String, String> entry: conf) {
System.out.printf("%s=%s\n", entry.getKey(), entry.getValue());
}
return 0;
}
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new ConfigPrinter(), args);
System.exit(exitCode);
}
}
生成jar,执行,打印环境结果。其中我加的-D参数会替换xml配置文件。
test2.xml放在src根目录下,或放到HADOOP_CONF_DIR能找到的路径中。
[zhouhh@Hadoop48 ~]$ hadoop jar ConfigPrinter.jar -Dmapred.map.tasks=6 io.map.index.skip=0 io.seqfile.compress.blocksize=1000000 mapred.task.profile.maps=0-2 keep.failed.task.files=false mapred.tasktracker.map.tasks.maximum=2 mapreduce.reduce.input.limit=-1 mapred.task.tracker.http.address=0.0.0.0:55060 mapred.map.tasks.speculative.execution=true mapred.used.genericoptionsparser=true mapred.userlog.retain.hours=24 webinterface.private.actions=false fs.s3.impl=org.apache.hadoop.fs.s3.S3FileSystem mapred.local.dir.minspacestart=0 hadoop.native.lib=true fs.checkpoint.edits.dir=${fs.checkpoint.dir} ipc.server.listen.queue.size=128 mapred.cluster.reduce.memory.mb=-1 io.sort.spill.percent=0.80 mapred.reduce.parallel.copies=5 tasktracker.http.threads=40 mapred.reduce.tasks=1 mapreduce.tasktracker.outofband.heartbeat=false hadoop.security.authorization=false io.file.buffer.size=4096 mapred.min.split.size=0 hadoop.logfile.size=10000000 mapred.job.queue.name=default mapred.submit.replication=10 mapred.local.dir.minspacekill=0 fs.webhdfs.impl=org.apache.hadoop.hdfs.web.WebHdfsFileSystem mapred.task.profile=false ipc.client.kill.max=10 mapred.acls.enabled=false mapred.heartbeats.in.second=100 mapreduce.reduce.shuffle.read.timeout=180000 mapred.output.compress=false ipc.server.tcpnodelay=false mapred.healthChecker.interval=60000 mapred.jobtracker.blacklist.fault-bucket-width=15 mapred.task.timeout=600000 mapred.temp.dir=${hadoop.tmp.dir}/mapred/temp mapred.jobtracker.taskScheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler mapred.max.tracker.blacklists=4 mapred.skip.reduce.max.skip.groups=0 mapred.tasktracker.indexcache.mb=10 mapreduce.jobtracker.staging.root.dir=${hadoop.tmp.dir}/mapred/staging mapred.queue.default.state=RUNNING mapred.tasktracker.dns.nameserver=default hadoop.logfile.count=10 mapred.tasktracker.taskmemorymanager.monitoring-interval=5000 mapred.tasktracker.expiry.interval=600000 hadoop.security.uid.cache.secs=14400 mapred.skip.attempts.to.start.skipping=2 mapreduce.reduce.shuffle.connect.timeout=180000 map.sort.class=org.apache.hadoop.util.QuickSort mapred.job.tracker.persist.jobstatus.active=false mapred.tasktracker.reduce.tasks.maximum=2 fs.ftp.impl=org.apache.hadoop.fs.ftp.FTPFileSystem mapred.max.tracker.failures=4 mapred.output.compression.codec=org.apache.hadoop.io.compress.DefaultCodec jobclient.output.filter=FAILED mapred.job.tracker.http.address=0.0.0.0:55030 fs.file.impl=org.apache.hadoop.fs.LocalFileSystem mapred.jobtracker.restart.recover=false mapred.healthChecker.script.timeout=600000 ipc.client.connection.maxidletime=10000 mapred.local.dir=${hadoop.tmp.dir}/mapred/local mapreduce.job.complete.cancel.delegation.tokens=true mapred.job.tracker.persist.jobstatus.dir=/jobtracker/jobsInfo mapred.job.tracker=Hadoop48:54311 io.sort.record.percent=0.05 job.end.retry.attempts=0 mapred.job.shuffle.merge.percent=0.66 mapred.map.max.attempts=4 mapred.reduce.tasks.speculative.execution=true mapreduce.job.counters.limit=120 fs.checkpoint.size=67108864 io.skip.checksum.errors=false mapred.job.reduce.input.buffer.percent=0.0 fs.s3n.impl=org.apache.hadoop.fs.s3native.NativeS3FileSystem fs.s3.maxRetries=4 mapred.output.compression.type=RECORD mapred.task.cache.levels=2 mapred.task.tracker.task-controller=org.apache.hadoop.mapred.DefaultTaskController mapred.job.reuse.jvm.num.tasks=1 mapred.system.dir=${hadoop.mydata.dir}/mapred/system test1=tdata1 io.sort.factor=10 mapred.userlog.limit.kb=0 mapred.jobtracker.maxtasks.per.job=-1 fs.default.name=hdfs://Hadoop48:54310 mapred.job.tracker.retiredjobs.cache.size=1000 ipc.client.idlethreshold=4000 fs.hsftp.impl=org.apache.hadoop.hdfs.HsftpFileSystem hadoop.tmp.dir=/tmp/hadoop-${user.name} fs.checkpoint.dir=${hadoop.tmp.dir}/dfs/namesecondary mapred.skip.map.auto.incr.proc.count=true fs.s3.block.size=67108864 io.serializations=org.apache.hadoop.io.serializer.WritableSerialization mapred.inmem.merge.threshold=1000 hadoop.util.hash.type=murmur io.seqfile.lazydecompress=true mapred.job.reduce.memory.mb=-1 mapred.skip.map.max.skip.records=0 mapred.job.map.memory.mb=-1 io.mapfile.bloom.size=1048576 hadoop.mydata.dir=/home/zhouhh/myhadoop fs.s3.buffer.dir=${hadoop.tmp.dir}/s3 mapred.tasktracker.dns.interface=default mapred.reduce.max.attempts=4 test2=tdata2 io.compression.codecs=org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec mapred.task.profile.reduces=0-2 mapred.job.tracker.jobhistory.lru.cache.size=5 mapred.cluster.map.memory.mb=-1 topology.script.number.args=100 mapred.skip.reduce.auto.incr.proc.count=true fs.har.impl=org.apache.hadoop.fs.HarFileSystem mapred.cluster.max.map.memory.mb=-1 mapred.job.tracker.persist.jobstatus.hours=0 io.seqfile.sorter.recordlimit=1000000 mapred.reduce.slowstart.completed.maps=0.05 fs.trash.interval=0 hadoop.security.authentication=simple local.cache.size=10737418240 hadoop.security.group.mapping=org.apache.hadoop.security.ShellBasedUnixGroupsMapping mapred.job.tracker.handler.count=10 hadoop.security.token.service.use_ip=true ipc.client.connect.max.retries=10 fs.ramfs.impl=org.apache.hadoop.fs.InMemoryFileSystem hadoop.rpc.socket.factory.class.default=org.apache.hadoop.net.StandardSocketFactory fs.kfs.impl=org.apache.hadoop.fs.kfs.KosmosFileSystem mapreduce.job.acl-view-job= fs.checkpoint.period=3600 mapred.map.output.compression.codec=org.apache.hadoop.io.compress.DefaultCodec topology.node.switch.mapping.impl=org.apache.hadoop.net.ScriptBasedMapping job.end.retry.interval=30000 mapred.tasktracker.tasks.sleeptime-before-sigkill=5000 mapred.job.shuffle.input.buffer.percent=0.70 mapred.jobtracker.completeuserjobs.maximum=100 mapred.user.jobconf.limit=5242880 mapred.compress.map.output=false mapred.queue.names=default fs.hdfs.impl=org.apache.hadoop.hdfs.DistributedFileSystem mapred.child.java.opts=-Xmx200m mapred.jobtracker.blacklist.fault-timeout-window=180 mapred.merge.recordsBeforeProgress=10000 mapred.jobtracker.job.history.block.size=3145728 mapreduce.reduce.shuffle.maxfetchfailures=10 io.mapfile.bloom.error.rate=0.005 mapreduce.job.split.metainfo.maxsize=10000000 io.bytes.per.checksum=512 mapred.child.tmp=./tmp fs.har.impl.disable.cache=true ipc.client.tcpnodelay=false fs.hftp.impl=org.apache.hadoop.hdfs.HftpFileSystem io.sort.mb=100 mapred.cluster.max.reduce.memory.mb=-1 mapred.line.input.format.linespermap=1 mapreduce.tasktracker.outofband.heartbeat.damper=1000000 mapreduce.job.acl-modify-job= mapred.combine.recordsBeforeProgress=10000 fs.s3.sleepTimeSeconds=10 mapred.map.tasks=6 mapred.task.tracker.report.address=127.0.0.1:0