这个类是用来跟Hadoop的文件系统进行交互的。尽管我们这里主要是针对HDFS。可是我们还是应该让我们的代码仅仅使用抽象类FileSystem。这样我们的代码就能够跟不论什么一个Hadoop的文件系统交互了。在写測试代码时,我们能够用本地文件系统測试,部署时使用HDFS。仅仅需配置一下,不须要改动代码了。
用Hadoop URL来读取HDFS里的文件
InputStream in = null;
try {
in = new URL("hdfs://host/path").openStream();
//操作输入流in。能够读取到文件的内容
} finally {
IOUtils.closeStream(in);
}
import java.io.InputStream;
import java.net.URL;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.io.IOUtils;
public class URLCat {
static {
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
}
public static void main(String[] args) throws Exception {
InputStream in = null;
try {
in = new URL(args[0]).openStream();
IOUtils.copyBytes(in, System.out, 4096, false);
} finally {
IOUtils.closeStream(in);
}
}
}
用FileSystem(org.apache.hadoop.fs.FileSystem)类来读取HDFS里的文件
public static FileSystem get(Configuration conf) throws IOException;
public static FileSystem get(URI uri, Configuration conf) throws IOException;
public static FileSystem get(final URI uri, final Configuration conf, final String user) throws IOException, InterruptedException;
public static LocalFileSystem getLocal(Configuration conf) throws IOException;
public FSDataInputStream open(Path f) throws IOException;
public abstract FSDataInputStream open(Path f, int bufferSize) throws IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
public class FileSystemCat {
public static void main(String[] args) throws Exception {
String uri = args[0];
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri), conf);
//System.out.println(fs.getClass().getName()); //这里能够看到得到的实例是DistributedFileSystem,由于core-site.xml里配的是hdfs
FSDataInputStream in = null;
try {
in = fs.open(new Path(uri));
IOUtils.copyBytes(in, System.out, 4096, false);
} finally {
IOUtils.closeStream(in);
}
}
}
in.seek(0);
IOUtils.copyBytes(in, System.out, 4096, false);
public int read(long position, byte[] buffer, int offset, int length) throws IOException;
public void readFully(long position, byte[] buffer, int offset, int length) throws IOException;
public void readFully(long position, byte[] buffer) throws IOException;
用FileSystem类来向HDFS里写文件
public FSDataOutputStream create(Path f) throws IOException;
public FSDataOutputStream create(Path f, Progressable progress) throws IOException;
public interface Progressable {
public void progress();
}
public FSDataOutputStream append(Path f) throws IOException;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
public class FileCopyWithProgress {
public static void main(String[] args) throws Exception {
String localSrc = args[0];
String dst = args[1];
InputStream in = new BufferedInputStream(new FileInputStream(localSrc));
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(dst), conf);
OutputStream out = fs.create(new Path(dst), new Progressable() {
@Override
public void progress() {
System.out.print(".");
// try {
// Thread.sleep(1000);
// } catch (Exception e) {
// e.printStackTrace();
// }
}
});
IOUtils.copyBytes(in, out, 4096, true);
System.out.println();
System.out.println(“end.”);
}
}
public long getPos() throws IOException;
创建文件夹
查询文件元信息:FileStatus(org.apache.hadoop.fs.FileStatus)
import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.*; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; import org.junit.Before; import org.junit.Test; public class ShowFileStatusTest { private static final String SYSPROP_KEY = "test.build.data"; /** MiniDFSCluster类在hadoop-hdfs-2.4.1-tests.jar中,是一个专门用于測试的in-process HDFS集群 */ private MiniDFSCluster cluster; private FileSystem fs; @Before public void setUp() throws IOException { Configuration conf = new Configuration(); String sysprop = System.getProperty(SYSPROP_KEY); if (sysprop == null) { System.setProperty(SYSPROP_KEY, "/tmp"); } cluster = new MiniDFSCluster(conf, 1, true, null); fs = cluster.getFileSystem(); OutputStream out = fs.create(new Path("/dir/file")); out.write("content".getBytes("UTF-8")); out.close(); } @After public void tearDown() throws IOException { if (fs != null) { fs.close(); } if (cluster != null) { cluster.shutdown(); } } @Test(expected = FileNotFoundException.class) public void throwsFileNotFoundForNonExistentFile() throws IOException { fs.getFileStatus(new Path("no-such-file")); } @Test public void fileStatusForFile() throws IOException { Path file = new Path("/dir/file"); FileStatus stat = fs.getFileStatus(file);
assertThat(stat.getPath().toUri().getPath(), is("/dir/file")); assertThat(stat.isDirectory(), is(false)); assertThat(stat.getLen(), is(7L)); assertTrue(stat.getModificationTime() <= System.currentTimeMillis()); assertThat(stat.getReplication(), is((short)1)); assertThat(stat.getBlockSize(), is(64 * 1024 * 1024L)); assertThat(stat.getOwner(), is("norris")); assertThat(stat.getGroup(), is("supergroup")); assertThat(stat.getPermission().toString(), is("rw-r--r--")); } @Test public void fileStatusForDirectory() throws IOException { Path dir = new Path("/dir"); FileStatus stat = fs.getFileStatus(dir); assertThat(stat.getPath().toUri().getPath(), is("/dir")); assertThat(stat.isDirectory(), is(true)); assertThat(stat.getLen(), is(0L)); assertTrue(stat.getModificationTime() <= System.currentTimeMillis()); assertThat(stat.getReplication(), is((short)0)); assertThat(stat.getBlockSize(), is(0L)); assertThat(stat.getOwner(), is("norris")); assertThat(stat.getGroup(), is("supergroup")); assertThat(stat.getPermission().toString(), is("rwxr-xr-x")); }
}