转:http://blog.csdn.net/uq_jin/article/details/52235121
1、解压Hadoop到任意目录
比如:D:\soft\dev\Hadoop-2.7.2
2、设置环境变量
HADOOP_HOME:D:\soft\dev\hadoop-2.7.2
HADOOP_BIN_PATH:%HADOOP_HOME%\bin
HADOOP_PREFIX:%HADOOP_HOME%
在Path后面加上%HADOOP_HOME%\bin;%HADOOP_HOME%\sbin;
把 hadoop-2.7.3\share\hadoop\hdfs\hadoop-hdfs-2.7.3.jar,share\hadoop\hdfs\lib
D:\BaiduYunDownload\hadoop-2.7.3\share\hadoop\common\hadoop-common-2.7.3.jar,\share\hadoop\common\lib
的jar包拷贝到项目lib下
HdfsClientTest
package com.hadoop.test;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;
public class HdfsClientTest {
private FileSystem fs = null;
@Before
public void setUp() throws Exception {
// get a configuration object
Configuration conf = new Configuration();
// to set a parameter,figure out the filesystemis hdfs
conf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");
conf.set("dfs.replication", "1");
// get a instance of HDFS FileSystem Client
fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");
}
@Test
public void testDownload() throws IllegalArgumentException, IOException {
FSDataInputStream is = fs.open(new Path("hdfs://192.168.169.128:9000/hadoop-common-2.7.3.jar"));
FileOutputStream os = new FileOutputStream("D:/hdfs/down.apk");
IOUtils.copy(is, os);
}
// 上传文件
public static void main(String[] args) throws Exception {
// get a configuration object
Configuration conf = new Configuration();
// to set a parameter,figure out the filesystemis hdfs
conf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");
conf.set("dfs.replication", "1");
// get a instance of HDFS FileSystem Client
FileSystem fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");
// open a outputstream of the dest file
Path destFile = new Path("hdfs://192.168.169.128:9000/hadoop-common-2.7.3.jar");
FSDataOutputStream os = fs.create(destFile);
// open a inputstream of the local source file
FileInputStream is = new FileInputStream(
"D:/BaiduYunDownload/hadoop-2.7.3/share/hadoop/common/hadoop-common-2.7.3.jar");
// write the bytes in "is" to "os"
IOUtils.copy(is, os);
}
}
HdfsClientEasy
package com.hadoop.test;
import static org.junit.Assert.*;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
public class HdfsClientEasy {
private FileSystem fs = null;
@Before
public void setUp() throws Exception {
// 拿到一个配置参数的封装对象,构造函数中就会对classpath下的xxx-site.xml文件进行解析
// 真实的项目工程中就应该把xxx-site.xml文件加入到工程中来
Configuration conf = new Configuration();
// to set a parameter,figure out the filesystemis hdfs
conf.set("fs.defalutFS", "hdfs://192.168.169.128:9000");
conf.set("dfs.replication", "1");
// 这种获取fs的方法可以指定访问hdfs的客户端身份
fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");
}
/**
* 上传文件
*
* @throws IllegalArgumentException
* @throws IOException
*/
@Test
public void testUpload() throws IllegalArgumentException, IOException {
fs.copyFromLocalFile(new Path("D:/HooShell_Shudu.apk"),
new Path("/hadoop-common-2.7.3.jar"));
}
/**
* 删除文件
*
* @throws IllegalArgumentException
* @throws IOException
*/
@Test
public void testRmfile() throws IllegalArgumentException, IOException {
boolean res = fs.delete(new Path("/hadoop-common-2.7.3.jar"), true);
System.out.println(res ? "delete is successfully :)" : "it is failed :(");
}
/**
* 创建文件夹
*
* @throws IllegalArgumentException
* @throws IOException
*/
@Test
public void testMkdir() throws IllegalArgumentException, IOException {
fs.mkdirs(new Path("/aa/bb"));
}
/**
* 重命名文件
*
* @throws IllegalArgumentException
* @throws IOException
*/
@Test
public void testRename() throws IllegalArgumentException, IOException {
fs.rename(new Path("/hadoop-common-2.7.3.jar"), new Path("/hadoop-common.jar"));
}
@Test
public void testListFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
while (listFiles.hasNext()) {
LocatedFileStatus file = listFiles.next();
System.out.println(file.getPath().getName());
}
System.out.println("--------------------------------------------");
// 列出文件及文件夹
FileStatus[] status = fs.listStatus(new Path("/"));
for (FileStatus file : status) {
System.out.println(file.getPath().getName() + " " + (file.isDirectory() ? "d" : "f"));
}
}
/**
* 从hdfs中下载数据到本地
*
* @throws IllegalArgumentException
* @throws IOException
*/
@Test
public void testDownload() throws IllegalArgumentException, IOException {
// fs.c
// fs.copyToLocalFile(true,new Path("/hadoop-common-2.7.3.jar"), new
// Path("d:/jdk12333.win"));
fs.copyToLocalFile(false, new Path("/hadoop-common-2.7.3.jar"), new Path("c:/jdk.win"), true);
}
}
出现的问题
Permission denied: user=root, access=WRITE, inode="/user/hive/warehouse":hadoop:hadoop:drwxrwxr-x
原因:本地用户administrator(本机windows用户)想要远程操作hadoop系统,没有权限引起的。
解决办法:
1、如果是测试环境,可以取消hadoop hdfs的用户权限检查。打开conf/hdfs-site.xml,找到dfs.permissions属性修改为false(默认为true)OK了。
2. 指定访问hdfs的客户端身份
fs = FileSystem.get(new URI("hdfs://192.168.169.128:9000/"), conf, "root");
fs.copyFromLocalFile(new Path("D:/HooShell_Shudu.apk"),new Path("/hadoop-common-2.7.3.jar"));报错
org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /HooShell_Shudu.apk could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and 1 node(s) are excluded in this operation.
at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1571)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getNewBlockTargets(FSNamesystem.java:3107)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3031)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:725)
解决方法:
#关闭防火墙
sudo systemctl stop firewalld.service
#关闭开机启动
sudo systemctl disable firewalld.service
java.io.IOException: (null) entry in command string: null chmod 0644 D:\hdfs\jdk123.win
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:769)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:866)
at org.apache.hadoop.util.Shell.execCommand(Shell.java:849)
at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:733)
at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:225)
at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.<init>(RawLocalFileSystem.java:209)
at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:307)
解决方法:将fs.copyToLocalFile( hdfsPath,localPath);改为fs.copyToLocalFile( false,hdfsPath,localPath,true);
原因:不理解,但加上这两个后确实可以下载