代码展示
public class HDFSUtil {
private FileSystem hdfs;
public HDFSUtil(String host, String port) {
this.hdfs = getFileSystem(host, port);
}
private static void kerberosConfig1(Configuration config) {
String krb5File = "D:\\data1\\krb5.conf";
String kerUser = "hdfs-geotmt@GAI.COM";
String keyPath = "D:\\data1\\hdfs.headless.keytab";
System.setProperty("java.security.krb5.conf", krb5File);
config.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(config);
try {
UserGroupInformation.loginUserFromKeytab(kerUser, keyPath);
} catch (IOException e) {
e.printStackTrace();
}
}
private static void kerberosConfig2(Configuration config) {
String krb5File = "D:\\data1\\krb5.conf";
String kerUser = "hdfs-geotmt@GAI.COM";
String keyPath = "D:\\data1\\hdfs.headless.keytab";
System.setProperty("java.security.krb5.conf", krb5File);
String coreSite = "D:\\data1\\core-site.xml";
String hdfsSite = "D:\\data1\\hdfs-site.xml";
System.setProperty("java.security.krb5.conf", krb5File);
config.addResource(new Path(coreSite));
config.addResource(new Path(hdfsSite));
UserGroupInformation.setConfiguration(config);
try {
UserGroupInformation.loginUserFromKeytab(kerUser, keyPath);
} catch (IOException e) {
e.printStackTrace();
}
}
public static FileSystem getFileSystem(String host, String port) {
Configuration config = new Configuration();
config.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
config.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
kerberosConfig1(config);
try {
URI uri = new URI("hdfs://" + host + ":" + port);
return FileSystem.get(uri, config);
} catch (IOException | URISyntaxException e) {
e.printStackTrace();
}
return null;
}
public void delete(String src) throws Exception {
Path p1 = new Path(src);
if (hdfs.isDirectory(p1)) {
hdfs.delete(p1, true);
System.out.println("删除文件夹成功: " + src);
} else if (hdfs.isFile(p1)) {
hdfs.delete(p1, false);
System.out.println("删除文件成功: " + src);
}
}
public void writerString(String text, String path) {
try {
Path f = new Path(path);
FSDataOutputStream os = hdfs.create(f, true);
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, "utf-8"));
writer.write(text);
writer.close();
os.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public List<String> readDirList(String dir) {
if (StringUtils.isBlank(dir)) {
return null;
}
if (!dir.startsWith("/")) {
dir = "/" + dir;
}
List<String> result = new ArrayList<String>();
FileStatus[] stats = null;
try {
stats = hdfs.listStatus(new Path(dir));
for (FileStatus file : stats) {
if (file.isFile() && file.getLen() != 0) {
result.add(file.getPath().toUri().getPath());
}
}
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
public void append(String data, String path) {
try {
Path f = new Path(path);
if (!hdfs.exists(f)) {
hdfs.createNewFile(f);
}
FSDataOutputStream os = hdfs.append(new Path(path));
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, "utf-8"));
writer.write(data);
writer.write(ToolUtil.lineSeparator());
writer.close();
os.close();
} catch (IOException e) {
e.printStackTrace();
}
}
public void uploadFileToHdfs(String data, String path) {
try {
data = new String(data.getBytes(), "utf-8");
this.writerString(data, path);
} catch (UnsupportedEncodingException e) {
throw new ServiceException(e.toString());
}
}
public static void main(String[] args) {
try {
HDFSUtil hdfsUtil = new HDFSUtil("10.111.32.184", "8020");
List<String> list = hdfsUtil.readDirList("/user/dp/file/data/temp/");
System.out.println("==============================================");
System.out.println(list);
System.out.println("==============================================");
} catch (Exception e) {
e.printStackTrace();
}
}
}