package com.test;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
public class Test {
public static void main(String args[]) throws Exception {
uploadToHdfs();
readFromHdfs();
}
//写入数据:从本机到云端(存储)
private static void uploadToHdfs() throws FileNotFoundException,IOException{
//我的文件地址
String localSrc = "F://hadoop/netdisk0.3/src/NetDisk.java";
//存放在云端的目的地址
String dest = "hdfs://127.0.0.1:9100/test/NetDisk112.java";
InputStream in = new BufferedInputStream(new FileInputStream(localSrc));
//得到配置对象
Configuration conf = new Configuration();
//文件系统
FileSystem fs = FileSystem.get(URI.create(dest), conf);
//输出流
OutputStream out = fs.create(new Path(dest), new Progressable() {
@Override
public void progress() {
System.out.println("上传完一个设定缓存区大小容量的文件!");
}
});
//连接两个流,形成通道,使输入流向输出流传输数据
IOUtils.copyBytes(in, out, 4096,true);
}
//读入数据:从云端到本机(提取数据)
private static void readFromHdfs()throws FileNotFoundException,IOException{
//云端地址
String dest = "hdfs://127.0.0.1:9100/test/NetDisk112.java";
//我的目录地址
String mySrc = "F://hadoop/netdisk0.3/NetDisk.java";
//得到配置
Configuration conf = new Configuration();
//实例化文件系统
FileSystem fs = FileSystem.get(URI.create(dest), conf);
//读出流
FSDataInputStream hdfsInStream = fs.open(new Path(dest));
//写入流
OutputStream out = new FileOutputStream(mySrc);
//将InputStrteam 中的内容通过IOUtils的copyBytes方法复制到out中
IOUtils.copyBytes(hdfsInStream, out, 4096,true);
}
}
其中String dest = "hdfs://127.0.0.1:9100/test/NetDisk112.java",hdfs://127.0.0.1:9100/为core-site.xml中的dfs配置,test为要上传的目录,NetDisk112.java为上传的文件名,注:至少要写一个上传的文件名,否则不能正常进行上传操作。
Unhandled exception. Starting shutdown.
java.lang.NoClassDefFoundError: org/apache/commons/configuration/Configuration解决办法:现在NoClassDefFoundError,缺少 org/apache/commons/configuration/Configuration从hadoop_home/lib下面cp一个commons-configuration-1.6.jar到项目的lib中。
package com.test;
import java.awt.FlowLayout;
import java.awt.TextField;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import javax.swing.Box;
import javax.swing.JButton;
import javax.swing.JFileChooser;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.JPanel;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
public class NetDisk extends JFrame{
public static String HDFS_Address = new String("hdfs://localhost:9100/tmp/hadoop-Administrator");
TextField file1 = new TextField(20);//C:\project\firsthadoop\src\WordCount.java
TextField file2 = new TextField(20);//hdfs://localhost:9000/user/Administrator/...
TextField file3 = new TextField(20);
JButton submit = new JButton("上传");
JButton chooser1 = new JButton("浏览");
JButton chooser2 = new JButton("浏览");
JButton chooser3 = new JButton("浏览");
JButton down = new JButton("下载");
JFileChooser jfc = new JFileChooser();
JPanel p1 = new JPanel();
JPanel p2 = new JPanel();
lisB btLis = new lisB();
/*
public NetDisk(){
setSize(500,400);
setTitle("网盘");
// p1.setLayout(new GridLayout(2,2));
chooser.addActionListener(new lisB());
p1.add(new JLabel("文件地址:"));
p1.add(file1);
p1.add(chooser);
p1.add(new JLabel("保存地址:"));
p1.add(file2);
add(p1,BorderLayout.NORTH);
submit.addActionListener(btLis);
down.addActionListener(btLis);
//add(submit,BorderLayout.CENTER);
p2.add(submit);
p2.add(down);
add(p2,BorderLayout.CENTER);
setVisible(true);
}
*/
public NetDisk(){
setSize(400,400);
setTitle("网盘");
Box basebox,box1,box2,box3;
chooser1.addActionListener(btLis);
chooser2.addActionListener(btLis);
chooser3.addActionListener(btLis);
file2.setText(HDFS_Address);
submit.addActionListener(btLis);
down.addActionListener(btLis);
box1 = Box.createVerticalBox();
box1.add(new JLabel("文件地址:"));
box1.add(Box.createHorizontalStrut(20));
box1.add(new JLabel("上传到:"));
box1.add(Box.createHorizontalStrut(20));
box1.add(new JLabel("下载到:"));
box2 = Box.createVerticalBox();
box2.add(file1);
box2.add(Box.createHorizontalStrut(20));
box2.add(file2);
box2.add(Box.createHorizontalStrut(20));
box2.add(file3);
box3 = Box.createVerticalBox();
box3.add(chooser1);
box3.add(Box.createHorizontalStrut(20));
box3.add(chooser2);
box3.add(Box.createHorizontalStrut(20));
box3.add(chooser3);
basebox = Box.createHorizontalBox();
basebox.add(box1);
basebox.add(Box.createHorizontalStrut(30));
basebox.add(box2);
basebox.add(Box.createHorizontalStrut(30));
basebox.add(box3);
setLayout(new FlowLayout());
add(basebox);
add(submit);
add(down);
setVisible(true);
}
class lisB implements ActionListener{
public void actionPerformed(ActionEvent e) {
FileSystem fs =null;
InputStream in = null;
OutputStream out = null;
String uri1 = file1.getText();//文件地址
String uri2 = file2.getText();//上传的位置
String uri3 = file3.getText();//下载的位置
Configuration conf = new Configuration();
if(e.getSource() == submit){
try {
in = new BufferedInputStream(new FileInputStream(uri1));
System.out.println("in is OK!");
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
try {
fs = FileSystem.get(URI.create(uri2),conf);
System.out.println("fs is OK!");
out = fs.create(new Path(uri2));
System.out.println("out is OK!");
IOUtils.copyBytes(in, out, 4096,true);
System.out.println("UPLoad is OK!");
} catch (IOException e1) {
e1.printStackTrace();
}
}
else if(e.getSource()==chooser1 || e.getSource() == chooser2 || e.getSource() == chooser3){
int state = jfc.showOpenDialog(null);
File file = jfc.getSelectedFile();
String s = file.getPath();
if(e.getSource() == chooser1)
file1.setText(s);
else if(e.getSource() == chooser2)
file2.setText(s);
else if(e.getSource() == chooser3)
file3.setText(s);
System.out.println(s);
}
else if(e.getSource() == down){
try {
fs = null;
fs = FileSystem.get(URI.create(uri2),conf);
in = null;
in = fs.open(new Path(uri2));
// String s = new String("C:\\hadoopdown\\1.jpeg");
// FileSystem fsout = FileSystem.get(URI.create(uri3), conf);
out = new BufferedOutputStream(new FileOutputStream(uri3));
IOUtils.copyBytes(in,out,4096,true);
System.out.println("down is OK!");
} catch (IOException e1) {
e1.printStackTrace();
}
}
}
}
public static void main(String[] args){
new NetDisk();
}
}
=========================
package com.test;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
public class ReadTest {
public static void main(String args[]) throws Exception {
uploadToHdfs();
// readFromHdfs();
}
//写入数据:从本机到云端(存储)
private static void uploadToHdfs() throws FileNotFoundException,IOException{
long s = System.currentTimeMillis();
//我的文件地址
// String localSrc = "F://hadoop/netdisk0.3/src/NetDisk.java";
String localSrc = "E://TDDOWNLOAD/记忆提取DVD中字.rmvb";
//存放在云端的目的地址
String dest = "hdfs://bida:9000/user/root/jytq.rmvb";
InputStream in = new BufferedInputStream(new FileInputStream(localSrc));
//得到配置对象
Configuration conf = new Configuration();
//文件系统
FileSystem fs = FileSystem.get(URI.create(dest), conf);
//输出流
OutputStream out = fs.create(new Path(dest), new Progressable() {
@Override
public void progress() {
System.out.println("上传完一个设定缓存区大小容量的文件!");
}
});
//连接两个流,形成通道,使输入流向输出流传输数据
IOUtils.copyBytes(in, out, 4096,true);
System.out.println((System.currentTimeMillis() - s)/1000+"s");
}
//读入数据:从云端到本机(提取数据)
private static void readFromHdfs()throws FileNotFoundException,IOException{
//云端地址
String dest = "hdfs://bida:9000/user/root/NetDisk112.java";
//我的目录地址
String mySrc = "F://hadoop/netdisk0.3/NetDisk.java";
//得到配置
Configuration conf = new Configuration();
//实例化文件系统
FileSystem fs = FileSystem.get(URI.create(dest), conf);
//读出流
FSDataInputStream hdfsInStream = fs.open(new Path(dest));
//写入流
OutputStream out = new FileOutputStream(mySrc);
//将InputStrteam 中的内容通过IOUtils的copyBytes方法复制到out中
IOUtils.copyBytes(hdfsInStream, out, 4096,true);
}
}
下面图片的问题为客户端访问服务端没有权限造成的,在hadoop的bin下执行hadoop fs -chmod 777 /root授权即可。
"main" org.apache.hadoop.security.AccessControlException: org.apache.hadoop.security.AccessControlException: Permission denied: user=Administrator, access=WRITE, inode="":root:supergroup:rwxr-xr-x