第1次在本地eclipse上写和运行hadoop程序,操作远程hadoop服务器。
遇到问题
1 java.io.IOException: Could not locate executable null\bin\winutils.exe in the Hadoop binaries.
这是没有配置环境变量HADOOP_HOME的原因,但是你可能会问,hadoop是装在服务器里,跟我本地有什么关系。跟踪代码,
发现一段这样的代码 ,String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" + File.separator + executable;
由于我是本地运行的,所以需要我本地的一些资源,所以还是设置下HADOOP_HOME环境变量,HADOOP_HOME指向的就是从官网下载的hadoop zip包解压路径,以我本地为例,我解压到D:\util\hadoop-2.7.2,那么HADOOP_HOME指向HADOOP_HOME。或者是加一段代码System.setProperty("hadoop.home.dir", "D:/util/hadoop-2.7.2");
但是修改后还是无法运行,%HADOOP_HOME%/bin/winutils.exe 也不存在,所以去下载一个,https://github.com/srccodes/hadoop-common-2.2.0-bin,下载下来,将winutils.exe 拷贝到%HADOOP_HOME%/bin/目录下。这个问题算是解决了
2 Wrong FS: hdfs://XXXXX expected: file:///
这个问题是由于,使用了默认的配置,在我的代码中,Configuration conf=new Configuration();没有指定任务参数,这样会使用jar包中默认的xml配置文件,各位可以解压jar包看看。要使程序连接到远程hdfs系统,就把服务器上core-site.xml和hdfs-site.xml拷贝过来,放到工程中,保证运行时这2个文件在classes目录下。
3 找不到master
配置本地host,使用SwitchHost工具,配置
192.168.8.97 master
192.168.8.98 slave1
192.168.8.119 slave2
代码:
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class HdfsTest {
private static Configuration conf;
private static FileSystem fs;
private static String fsPath="hdfs://master:9000/user/lwq/";
static {
System.setProperty("hadoop.home.dir", "D:/util/hadoop-2.7.2");
try {
conf=new Configuration();
fs = FileSystem.get(conf);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void deleteFile(){
Path path=new Path(fsPath+"a.log");
try {
fs.delete(path);
fs.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
//读取文件block所在地方
public static void getFileBlockLocation(){
try{
Path f = new Path(fsPath+"a.log");
FileStatus fileStatus = fs.getFileStatus(f);
BlockLocation[] blkLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation currentLocation : blkLocations) {
String[] hosts = currentLocation.getHosts();
for (String host : hosts) {
System.out.println("该文件存在于node:"+host);
}
}
}catch(Exception e){
e.printStackTrace();
}
}
public static void read(){
try {
Path listf = new Path(fsPath);
FileStatus stats[] = fs.listStatus(listf);
InputStream in;
BufferedReader buff = null;
for (int i = 0; i < stats.length; i++) {
String url = stats[i].getPath().toString();
System.out.println(url);
FileStatus temp[] = fs.listStatus(new Path(stats[i].getPath().toString()));
for (int k = 0; k < temp.length; k++) {
System.out.println("文件路径名:" + temp[k].getPath().toString());
if(temp[k].isDir()){
System.out.println("该路径为目录");
continue;
}else{
System.out.println("该路径为文件");
}
Path p = new Path(temp[k].getPath().toString());
in = fs.open(p);
buff = new BufferedReader(new InputStreamReader(in));
String str = null;
while ((str = buff.readLine()) != null) {
System.out.println(str);
}
buff.close();
in.close();
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String args[]){
// read
getFileBlockLocation();
// deleteFile
}
}