环境配置
1. 将hadoop解压到window中,路径不能有中文,不能有空格。
2. 拷贝hadoop的windows执行环境工具到bin路径下,替换客户端工具。
3. 配置环境变量
HADOOP_HOME=hadoop安装路径
PATH=hadoop安装路径/bin
4. 重启IDEA或者eclipse
Hadoop项目maven依赖
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.3.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
<!-- log4j依赖 -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.17.2</version>
</dependency>
</dependencies>
增加日志配置文件
Log4j 支持两种配置文件格式,一种是 XML 格式的文件,一种是 Java 特性文件 lg4j.properties (键 = 值)
log4j中的xml配置文件要优先于properties文件
(1)log4j2.xml
在 src/main/resources 目录中增加日志配置文件 log4j2.xml
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="error" strict="true" name="XMLConfig">
<Appenders>
<Appender type="Console" name="STDOUT">
<Layout type="PatternLayout"
pattern="[%p] [%d{yyyy-MM-dd HH:mm:ss}][%c{10}]%m%n" />
</Appender>
</Appenders>
<Loggers>
<!-- 可加性为false -->
<Logger name="test" level="info" additivity="false">
<AppenderRef ref="STDOUT" />
</Logger>
<!-- root loggerConfig设置 -->
<Root level="info">
<AppenderRef ref="STDOUT" />
</Root>
</Loggers>
</Configuration>
(2)lg4j.properties
### 设置###
log4j.rootLogger = debug,stdout,D,E
### 输出信息到控制抬 ###
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
### 输出DEBUG 级别以上的日志到 ###
log4j.appender.D = org.apache.log4j.DailyRollingFileAppender
log4j.appender.D.File = D://linuxvmware/hadooptest2/src/main/resources/data/log.log
log4j.appender.D.Append = true
log4j.appender.D.Threshold = DEBUG
log4j.appender.D.layout = org.apache.log4j.PatternLayout
log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
### 输出ERROR 级别以上的日志到=error.log ###
log4j.appender.E = org.apache.log4j.DailyRollingFileAppender
log4j.appender.E.File =D://linuxvmware/hadooptest2/src/main/resources/data/error.log
log4j.appender.E.Append = true
log4j.appender.E.Threshold = ERROR
log4j.appender.E.layout = org.apache.log4j.PatternLayout
log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
执行代码
System.setProperty("HADOOP_USER_NAME","root");
Configuration conf = new Configuration();
conf.set("fs.defaultFS","hdfs://192.168.244.100:8020");
conf.set("dfs.client.use.datanode.hostname", "true");
FileSystem fs = FileSystem.get(conf);
//创建目录
Boolean flag = fs.mkdirs(new Path("/root/data"));
System.out.println(flag);
//判断文件是否存在
Boolean isExists = fs.exists(new Path("/root/data/student2.txt"));
System.out.println(isExists);
if(isExists == false){
fs.create(new Path("/root/data/student2.txt"));
}
//查看文件信息
FileStatus fileStatus = fs.getFileStatus(new Path("/root/data/student2.txt"));
System.out.println(fileStatus.toString());
//下载文件
if(isExists == false){
fs.copyToLocalFile(new Path("/root/data/"),new Path("D:\\linuxvmware\\hadooptest2\\src\\main\\resources\\"));
}
//上传文件
String srcFilePath = "C:\\Users\\86132\\Desktop\\0715.txt";
String remoteFilePath = "/root/data/";
fs.copyFromLocalFile(new Path(srcFilePath),new Path(remoteFilePath));
//删除文件
//fs.delete(new Path("/root/data/"),true);
//查看文件列表
RemoteIterator<LocatedFileStatus> fileList = fs.listFiles(new Path("/root/data/"), true);
while (fileList.hasNext()){
System.out.println(fileList.next().toString());
}
Google Option概要及使用
安装
<dependency>
<groupId>com.github.pcj</groupId>
<artifactId>google-options</artifactId>
<version>1.0.0</version>
</dependency>
创建扩展和定义的解析器类
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionsBase;
import java.util.List;
/**
* Command-line options definition for example server.
*/
public class ServerOptions extends OptionsBase {
@Option(
name = "help",
abbrev = 'h',
help = "Prints usage info.",
defaultValue = "true"
)
public boolean help;
@Option(
name = "host",
abbrev = 'o',
help = "The server host.",
category = "startup",
defaultValue = ""
)
public String host;
@Option(
name = "port",
abbrev = 'p',
help = "The server port.",
category = "startup",
defaultValue = "8080"
)
public int port;
@Option(
name = "dir",
abbrev = 'd',
help = "Name of directory to serve static files.",
category = "startup",
allowMultiple = true,
defaultValue = ""
)
public List<String> dirs;
}
解析器类的参数使用
import com.google.devtools.common.options.OptionsParser;
import java.util.Collections;
public class Server {
//java -jar server.jar --host=127.0.0.1 --port=21 --dir=E:\test --dir=E:\demo
public static void main(String[] args) {
OptionsParser parser = OptionsParser.newOptionsParser(ServerOptions.class);
parser.parseAndExitUponError(args);
ServerOptions options = parser.getOptions(ServerOptions.class);
if (options.host.isEmpty() || options.port < 0 || options.dirs.isEmpty()) {
printUsage(parser);
return;
}
System.out.format("Starting server at %s:%d...\n", options.host, options.port);
for (String dirname : options.dirs) {
System.out.format("\\--> Serving static files at <%s>\n", dirname);
}
}
private static void printUsage(OptionsParser parser) {
System.out.println("Usage: java -jar server.jar OPTIONS");
System.out.println(parser.describeOptions(Collections.<String, String>emptyMap(),
OptionsParser.HelpVerbosity.LONG));
}
}