Hadoop_day04学习笔记

直接新建一个Java工程

通过java去读取hdfs 

  

抛出异常  原因是java无法识别hdfs协议   上面传的8020是网络端口

导入jar包之后 就可以识别hdfs协议了

抛出警告  log4j 把hadoopjar包share下的log4j.properties复制到src目录下面即可

之后就是通过hdfs API的一系列操作

1.读取文件

2.创建目录

3.创建文件

4.上传文件

5.删除文件

6.删除目录

7.列出目录下的文件

需要注意的就是在创建目录时涉及到权限问题 会抛出异常 赋予要写的目录W(写)权限就可以

代码

package com.shulian.hadoop_day04;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class Hadoop01 {
	public static void main(String[] args) throws Exception {
		listfile();
	}
	/**
	 * 通过java去读HDFS文件
	 * @throws Exception 
	 */
	public static void read() throws Exception{
		URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
		URL url = new URL("hdfs://192.168.163.101:8020/user/hadoop/core-site.xml");
		URLConnection openConnection = url.openConnection();
		InputStream inputstream = openConnection.getInputStream();
		byte[] bys = new byte[1024];
		int len = 0;
		while((len=inputstream.read(bys))!=-1){
			System.out.println(new String(bys,0,len));
		}
		inputstream.close();
	}
	/**
	 * 通过hdfs读取hdfs文件
	 * @throws IOException 
	 */
	public static void read2() throws IOException{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");//这里是core文件里面的
		FileSystem file = FileSystem.get(conf);
		Path path = new Path("/user/hadoop/core-site.xml");
		//open方法打开文件
		FSDataInputStream open = file.open(path);
		ByteArrayOutputStream bys = new ByteArrayOutputStream();
		byte[] buf = new byte[1024];
		int len = -1 ; 
		while((len=open.read(buf)) !=-1){
			bys.write(buf,0,len);
		}
		System.out.println(new String(bys.toByteArray()));
		open.close();
		bys.close();
	}
	/**
	 * 用hdfs API读取文件  用了一个工具类  IOUtils.copyBytes(open, bys, 1024);
	 * @throws IOException
	 */
	public static void read3() throws IOException{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");//这里是core文件里面的
		FileSystem file = FileSystem.get(conf);
		Path path = new Path("/user/hadoop/core-site.xml");
		//open方法打开文件
		FSDataInputStream open = file.open(path);
		ByteArrayOutputStream bys = new ByteArrayOutputStream();
		IOUtils.copyBytes(open, bys, 1024);
		System.out.println(new String(bys.toByteArray()));
		open.close();
		bys.close();
	}
	/**
	 * 使用用户 hdfs API创建目录
	 * @throws Exception
	 */
	public static void mkdir() throws Exception{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
		FileSystem file = FileSystem.get(conf);
		Path path = new Path("/user/hadoop/newdir");
		boolean mkdirs = file.mkdirs(path);
		System.out.println(mkdirs);
	}
	/**
	 * 使用hdfs API创建文件
	 * @throws IOException
	 */
	public static void createfile() throws Exception{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
		FileSystem file = FileSystem.get(conf);
		Path path = new Path("/user/hadoop/newdir/newfile.txt");
		FSDataOutputStream create = file.create(path);
		create.writeBytes("hello world");
		create.close();
	}
	/**
	 * 通过hdfs API上传文件
	 * @throws Exception
	 */
	public static void copyfile() throws Exception{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
		FileSystem file = FileSystem.get(conf);
		Path src = new Path("F:\\work\\zijilianxi_16\\src\\com\\dashuju");
		Path dst = new Path("/user/hadoop/");
		file.copyFromLocalFile(src, dst);
	}
	/**
	 * 通过hdfs API 删除文件
	 * @throws Exception
	 */
	public static void deletefile() throws Exception{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
		FileSystem file = FileSystem.get(conf);
		Path path = new Path("/user/hadoop/newdir/newfile.txt");
		boolean delete = file.delete(path,true);
		System.out.println(delete);
	}
	/**
	 * 通过hdfs API删除目录
	 * @throws Exception
	 */
	public static void deletedir() throws Exception{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
		FileSystem file = FileSystem.get(conf);
		Path path = new Path("/user/hadoop/newdir");
		boolean delete = file.delete(path,true);
		System.out.println(delete);
	}
	/**
	 * 通过hdfs 列出目录下所有文件即目录
	 * @throws Exception
	 */
	public static void listfile() throws Exception{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
		FileSystem file = FileSystem.get(conf);
		Path path = new Path("/user/hadoop");
		FileStatus[] listStatus = file.listStatus(path);
		for(FileStatus f :listStatus){
			System.out.println(f.getPath().getName());
		}
	}
}

通过maven就不需要导入jar包了  弄一下pom.xml依赖关系就可以

还配置了一个hadoop变量在windows上  不配置的话还抛出异常

还有一个递归列出所有的目录文件

import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class Hdfs {
	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
		FileSystem filesystem = FileSystem.get(conf);
		Path path = new Path("/");
		digui(filesystem,path);
	}
	public static void digui(FileSystem f, Path path) throws Exception {
		FileStatus[] listStatus = f.listStatus(path);
		for (FileStatus file : listStatus) {
			if (file.isFile()) {
				System.out.println("文件名:"+file.getPath().getName()+"\t"+"全路径"+file.getPath().getParent()+"\t"+"所属用户"+file.getOwner()+"\t"+"时间"+shijian(file.getModificationTime())+"\t"+"大小"+file.getLen());
			} else {
				System.out.println("文件夹名："+file.getPath().getName()+"\t"+"全路径"+file.getPath().getParent()+"\t"+"所属用户"+file.getOwner()+"\t"+"时间"+shijian(file.getModificationTime()));
				Configuration conf = new Configuration();
				conf.set("fs.defaultFS", "hdfs://192.168.163.101:8020/");
				FileSystem fss = FileSystem.get(conf);
				digui(fss, file.getPath());
			}
		}
	}
	public static String shijian(long s){
		String pa = "yyyy-MM-dd HH:mm:ss";
		SimpleDateFormat j = new SimpleDateFormat(pa);
		Date dd = new Date(s);
		
		return j.format(dd);
	}
}