学习hadoop的hdfs开发(java)

• 测试代码

package net.xxx;

import java.io.IOException;

import java.net.URI;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FSDataOutputStream;

public class test {

 public void WriteFile(String hdfs) throws IOException {

  Configuration conf = new Configuration();

  try{

  FileSystem fs = FileSystem.get(URI.create(hdfs),conf,"hdfs");

  FSDataOutputStream hdfsOutStream = fs.create(new Path(hdfs));

  hdfsOutStream.writeChars("hello");

  hdfsOutStream.close();

  fs.close();

  }catch(Exception e){

    System.out.println(e);

  }
 }

 public void ReadFile(String hdfs) throws IOException {
  Configuration conf = new Configuration();
  try{
  FileSystem fs = FileSystem.get(URI.create(hdfs),conf,"hdfs");
  FSDataInputStream hdfsInStream = fs.open(new Path(hdfs));

  byte[] ioBuffer = new byte[1024];
  int readLen = hdfsInStream.read(ioBuffer);
  while(readLen!=-1)
  {
   System.out.write(ioBuffer, 0, readLen);
   readLen = hdfsInStream.read(ioBuffer);
  }
  System.out.println();

  hdfsInStream.close();
  fs.close();
  }catch(Exception e){
    System.out.println(e);
  }
 }

 public static void main(String[] args) throws IOException {
  String hdfs = "hdfs://test127:8020/test/hello.txt";
  test t = new test();
  t.WriteFile(hdfs);
  t.ReadFile(hdfs);
 }
}

• 依赖库编译

cp "/opt/cm-5.12.1/share/cmf/common_jars/hadoop-common-2.5.0-cdh5.3.3.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/lib/commons-logging-1.1.1.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/guava-14.0.1.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/commons-collections-3.2.2.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/commons-configuration-1.9.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/commons-lang-2.6.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/hadoop-auth-2.5.0-cdh5.3.3.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/slf4j-log4j12-1.7.5.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/slf4j-api-1.7.5.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/log4j-1.2.17.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/common_jars/hadoop-hdfs-2.5.0-cdh5.3.3.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/cloudera-navigator-server/libs/cdh5/servlet-api-2.5.jar" ./lib
cp "/opt/cm-5.12.1/share/cmf/cloudera-navigator-server/libs/cdh5/protobuf-java-2.5.0.jar" ./lib

• 编译

javac test.java -cp "./lib/hadoop-common-2.5.0-cdh5.3.3.jar"

• manifest.mf

Manifest-Version: 1.0

Class-Path: lib/hadoop-hdfs-2.5.0-cdh5.3.3.jar lib/hadoop-common-2.5.0-cdh5.3.3.jar lib/commons-logging-1.1.1.jar lib/guava-14.0.1.jar lib/commons-collections-3.2.2.jar lib/commons-configuration-1.9.jar lib/commons-lang-2.6.jar lib/hadoop-auth-2.5.0-cdh5.3.3.jar lib/slf4j-log4j12-1.7.5.jar lib/slf4j-api-1.7.5.jar lib/log4j-1.2.17.jar lib/hadoop-hdfs-2.5.0-cdh5.3.3.jar lib/servlet-api-2.5.jar lib/commons-cli-1.4.jar lib/protobuf-java-2.5.0.jar

Main-Class: net.xxx.test

• 打包

jar cvfm test.jar manifest.mf lib net

• 运行

java -jar test.jar