hadoop cloudera CDH5.13.1 java编程向HDFS写入文件
1.以用户hdfs身份创建需写入文件的目录
root@cdh01:~#su hdfshadoop fs -mkdir /output
hadoop fs -chmod 777 /output
exit
2. 创建目录 /root/java 存放编写及编译的代码
root@cdh01:~# mkdir java
root@cdh01:~# cd java
root@cdh01:~/java# vi HDFSCreateFile.java
编辑文件名为 HDFSCreateFile.java 内容如下
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HDFSCreateFile {
public static void main(String[] args){
try{
Configuration conf = new Configuration();
conf.set("fs.defaultFS","hdfs://cdh01:8020");
conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
FileSystem fs = FileSystem.get(conf);
byte[] buff = "Hello World!".getBytes();
String fileName = "/output/test.txt";
FSDataOutputStream os =fs.create(new Path(fileName));
os.write(buff,0,buff.length);
System.out.println("Create:"+fileName);
os.close();
fs.close();
} catch (Exception e){
e.printStackTrace();
}
}
}
3.编译java源文件
root@cdh01:~/java# javac -Djava.ext.dirs=/opt/cloudera/parcels/CDH/jars HDFSCreateFile.java
注意指定包含的第三方jar包所在目录
编译后,目录下可以看到HDFSCreateFile.class 文件
4. 创建manifest.txt
# echo Main-class: HDFSCreateFile > manifest.txt
该文件指定主程序入口
5.生成可执行jar文件
jar cvfm HDFSCreateFile.jar manifest.txt HDFSCreateFile.class
6.测试执行文件
root@cdh01:~/java# hadoop jar HDFSCreateFile.jar
7.查看程序建立的文件
hadoop fs -cat /output/test.txt