大数据之Hadoop02-java客户端操作hdfs及hdfs原理增强

java客户端操作hdfs

在java客户端操作hdp需要先创建一个Maven项目,并导入如下依赖:

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>doit17_hdp</artifactId>
    <version>1.0-SNAPSHOT</version>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>3.2.1</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-common</artifactId>
            <version>3.2.1</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.68</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
            <version>3.2.1</version>
        </dependency>
    </dependencies>

</project>

**
 *(1)java客户端操作hdfs,上传文件,删除文件,创建文件夹,
 */
public class Demo01 {
    public static void main(String[] args) {
        //配置信息对象
        Configuration conf = new Configuration();
        conf.set("dfs.replication","4");
        FileSystem fs = null;
        try {
            //获取hdfs对象
            fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf, "root");
            //上传文件的方法,类似-put
            //fs.copyFromLocalFile(new Path("D:\\anzhuangbao\\apache-maven-3.6.3-bin.zip"),new Path("/data"));
            //删除文件的方法,true表示递归删除
            fs.delete(new Path("/data/apache-maven-3.6.3-bin.zip"),true);
            //创建文件夹
            //fs.mkdirs(new Path("/data02"));
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                fs.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

/**
 * (2)使用配置文件设定切块大小
 */
//先在resoures文件夹中书写hdfs-site.xml的配置文件
<configuration>
	<property>
	     <name>dfs.blocksize</name>
		 <value>64M</value>
	</property>
	<property>
		<name>dfs.replication</name>
		<value>2</value>
	</property>
</configuration>

public class Demo02 {
    public static void main(String[] args) throws Exception {
    //new Configuration对象时,会自动去配置文件中读取配置的信息
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.newInstance(new URI("hdfs://linux01:8020"), conf,"root");
        fs.copyFromLocalFile(new Path("D:\\anzhuangbao\\HBuilder.9.1.29.windows.zip"),new Path("/a.zip3"));
        fs.close();
    }
}

/**
 * (3)封装一个获取fs对象的方法
 */
public class FileSystemUtils {
    public static FileSystem getfs() throws Exception {
        Configuration conf = new Configuration();
       return FileSystem.newInstance(new URI("hdfs://linux01:8020"),conf,"root");
    }
}

/**
 * (4)从hdfs下载文件到windows
 */
public class Demo03 {
    public static void main(String[] args) throws Exception {
        FileSystem fs = FileSystemUtils.getfs();
        /*
        参数一:是否删除源文件
        参数二:hdfs文件路径
        参数三:本地下载路径
        参数四:是否生成校验文件,true表示不生成
         */
        fs.copyToLocalFile(true,new Path("/HBuilder.9.1.29.windows.zip"),new Path("d:\\"),true);
    }
}

    /**
     * (5)使用四参数上传文件的方法
     */
    public static void main(String[] args) throws Exception {
        FileSystem fs = FileSystemUtils.getfs();
        /*
        参数一:是否删除源文件
        参数二:是否覆盖同名文件
        参数三:本地想上传的文件路径
        参数四:hdfs接收的文件路径
         */
        fs.copyFromLocalFile(true, true, new Path("D:\\HBuilder.9.1.29.windows.zip") ,new Path("/"));
    }
}

/**
 * (6)遍历文件夹下所有子文件的信息
 */
public class ListFilesDemo {
    public static void main(String[] args) throws Exception {
        //获取hdfs连接对象
        FileSystem fs = FileSystemUtils.getfs();
        //获取需要遍历的文件路径
        Path path = new Path("/");
        //判断改路径是否是一个文件夹
        FileStatus fils = fs.getFileStatus(path);
        boolean dir = fils.isDirectory();
        if (dir) {
            //迭代遍历文件夹,返回一个迭代器
            RemoteIterator<LocatedFileStatus> ls = fs.listFiles(path, true);
            while (ls.hasNext()) {
                //获取到每个文件的对象
                LocatedFileStatus next = ls.next();
                //获取文件大小
                long len = next.getLen();
                //获取文件路径
                Path path1 = next.getPath();
                //获取副本个数
                short replication = next.getReplication();
                System.out.println(path1+"--"+len/1024/1024+"M");
                //获取到文件的块对象,返回一个数组,遍历此数组
                BlockLocation[] blockLocations = next.getBlockLocations();
                for (BlockLocation b : blockLocations) {
                    //获取块名
                    b.getNames();
                    //获取块的偏移量
                    long offset = b.getOffset();
                    //获取块大小
                    long length = b.getLength();
                    System.out.println(offset/1024/1024+"M--"+length/1024/1024+"M");
                    //获取该块的主机名,返回一个数组,遍历该数组
                    String[] hosts = b.getHosts();
                    for (String host : hosts) {
                        System.out.println(host);
                    }
                }
            }
        }
        fs.close();
    }
}

/**
 * (7)读hdfs中的文件
 */
public class ReadDemo {

    public static void main(String[] args) throws Exception {
        //获取hdfs的连接对象
        FileSystem fs = FileSystemUtils.getfs();
        //使用分布式流读取
        FSDataInputStream open = fs.open(new Path("/data/1.txt"));
        open.seek(10);//跳过10个字节
        //转换为缓冲字节流读取
        BufferedReader br = new BufferedReader(new InputStreamReader(open));
        String line = null;
        while ((line = br.readLine())!=null){
            System.out.println(line);
        }
        open.close();
        br.close();
        fs.close();
    }
}

 	 /**
     *(8) 往hdfs里写
     */
public class WriteDemo {
    public static void main(String[] args) throws Exception {
        FileSystem fs = FileSystemUtils.getfs();
        //append()追加写入
        //FSDataOutputStream f = fs.append(new Path("/data/1.txt")); 
        //create()覆盖写入,文件不存在时,会创建文件再写入
        FSDataOutputStream f = fs.create(new Path("/data/1.txt"));
        f.writeUTF("asdasscascw");
        fs.close();
    }
}

nemenode和datanode之间通信导图

Hadoop写数据流程导图

Hadoop下载数据流程导图