centos6.4 使用 HDFS C API

最新推荐文章于 2023-08-26 07:45:00 发布

wind520

最新推荐文章于 2023-08-26 07:45:00 发布

阅读量1.7k

点赞数

分类专栏： Hadoop Linux

本文链接：https://blog.csdn.net/wind520/article/details/38402261

版权

Linux 同时被 2 个专栏收录

42 篇文章 0 订阅

订阅专栏

Hadoop

30 篇文章 0 订阅

订阅专栏

centos6.4 32位

1：配置HADOOP_HOME

[jifeng@jifeng02 HDFS_C_API]$ vi /home/jifeng/.bash_profile


# .bash_profile

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
        . ~/.bashrc
fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH
export JAVA_HOME=$HOME/jdk1.7.0_45
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export HADOOP_HOME=$HOME/hadoop/hadoop-1.2.1
export ANT_HOME=$HOME/apache-ant-1.9.4

export PATH=$PATH:$ANT_HOME/bin
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
~
"~/.bash_profile" 19L, 427C 已写入

2：C的代码

[jifeng@jifeng02 HDFS_C_API]$ cat hdfs_cpp_demo.c

// Following is a libhdfs sample adapted from the src/c++/libhdfs/hdfs_write.c of the Hadoop distribution.

#include "hdfs.h" 

int main(int argc, char **argv) {
    
    hdfsFS fs = hdfsConnect("jifeng01",9000);
    if (!fs) {
        fprintf(stderr, "Cannot connect to HDFS.\n");
        exit(-1);
    } 
 
    char* fileName = "demo_c.txt";
    char* message = "Welcome to HDFS C API!!!";
    int size = strlen(message);
    
    int exists = hdfsExists(fs, fileName);

    if (exists > -1) {
        fprintf(stdout, "File %s exists!\n", fileName);
    }else{
        // Create and open file for writing
        hdfsFile outFile = hdfsOpenFile(fs, fileName, O_WRONLY|O_CREAT, 0, 0, 0);
        if (!outFile) {
            fprintf(stderr, "Failed to open %s for writing!\n", fileName);
            exit(-2);
        }
        // write to file
        hdfsWrite(fs, outFile, (void*)message, size); 
        hdfsCloseFile(fs, outFile);
    }
    
    // Open file for reading
    hdfsFile inFile = hdfsOpenFile(fs, fileName, O_RDONLY, 0, 0, 0);
    if (!inFile) {
        fprintf(stderr, "Failed to open %s for reading!\n", fileName);
        exit(-2);
    }

    char* data = malloc(sizeof(char) * size); 
    // Read from file. 
    tSize readSize = hdfsRead(fs, inFile, (void*)data, size);
    fprintf(stdout, "%s\n", data);
    free(data);

    hdfsCloseFile(fs, inFile);
    hdfsDisconnect(fs);
    return 0;
}

3：编译

32编译

[jifeng@jifeng02 HDFS_C_API]$ gcc hdfs_cpp_demo.c -I $HADOOP_HOME/src/c++/libhdfs -I $JAVA_HOME/include -I $JAVA_HOME/include/linux -L $HADOOP_HOME/c++/Linux-i386-32/lib/  -L $JAVA_HOME/jre/lib/i386/server -l hdfs -ljvm -o hdfs_cpp_demo
[jifeng@jifeng02 HDFS_C_API]$ ls
build.xml  hdfs_cpp_demo  hdfs_cpp_demo.c
[jifeng@jifeng02 HDFS_C_API]$

64位编译

[jifeng@jifeng04 HDFS_C_API]$ gcc hdfs_cpp_demo.c \
> -I $HADOOP_HOME/src/c++/libhdfs \
> -I $JAVA_HOME/include \
> -I $JAVA_HOME/include/linux/  \
> -L $HADOOP_HOME/c++/Linux-amd64-64/lib/ -lhdfs \
> -L $JAVA_HOME/jre/lib/amd64/server  -ljvm \
> -o hdfs_cpp_demo
[jifeng@jifeng04 HDFS_C_API]$ gcc hdfs_cpp_demo.c -I $HADOOP_HOME/src/c++/libhdfs -I $JAVA_HOME/include -I $JAVA_HOME/include/linux/  -L $HADOOP_HOME/c++/Linux-amd64-64/lib/ -lhdfs -L $JAVA_HOME/jre/lib/amd64/server  -ljvm -o hdfs_cpp_demo
[jifeng@jifeng04 HDFS_C_API]$ ls
build.xml  hdfs_cpp_demo  hdfs_cpp_demo.c
[jifeng@jifeng04 HDFS_C_API]$

4:运行

[jifeng@jifeng04 HDFS_C_API]$ LD_LIBRARY_PATH=$HADOOP_HOME/c++/Linux-amd64-64/lib:$JAVA_HOME/jre/lib/amd64/server ./hdfs_cpp_demo
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configuration
        at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
        at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
        at java.security.AccessController.doPrivileged(Native Method)
        at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
Can't construct instance of class org.apache.hadoop.conf.Configuration
Cannot connect to HDFS.
[jifeng@jifeng04 HDFS_C_API]$ ls
build.xml  hdfs_cpp_demo  hdfs_cpp_demo.c

[jifeng@jifeng04 ~]$ ant -version
Apache Ant(TM) version 1.9.4 compiled on April 29 2014

[jifeng@jifeng04 HDFS_C_API]$ ls
build.xml  hdfs_cpp_demo  hdfs_cpp_demo.c
[jifeng@jifeng04 HDFS_C_API]$ ant print-cp
Buildfile: /home/jifeng/hadoop/HDFS_C_API/build.xml

print-cp:
     [echo] classpath= /home/jifeng/hadoop/hadoop-1.2.1/lib/asm-3.2.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/aspectjrt-1.6.11.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/aspectjtools-1.6.11.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-beanutils-1.7.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-beanutils-core-1.8.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-cli-1.2.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-codec-1.4.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-collections-3.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-configuration-1.6.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-daemon-1.0.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-digester-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-el-1.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-httpclient-3.0.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-io-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-lang-2.4.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-logging-1.1.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-logging-api-1.0.4.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-math-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-net-3.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/core-3.1.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hadoop-capacity-scheduler-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hadoop-fairscheduler-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hadoop-thriftfs-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hsqldb-1.8.0.10.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jackson-core-asl-1.8.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jackson-mapper-asl-1.8.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jasper-compiler-5.5.12.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jasper-runtime-5.5.12.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jdeb-0.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jersey-core-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jersey-json-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jersey-server-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jets3t-0.6.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jetty-6.1.26.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jetty-util-6.1.26.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jsch-0.1.42.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jsp-2.1/jsp-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jsp-2.1/jsp-api-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/junit-4.5.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/kfs-0.2.2.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/log4j-1.2.15.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/mockito-all-1.8.5.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/oro-2.0.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/servlet-api-2.5-20081211.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/slf4j-api-1.4.3.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/slf4j-log4j12-1.4.3.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/xmlenc-0.52.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/datajoin/hadoop-datajoin-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/failmon/hadoop-failmon-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/gridmix/hadoop-gridmix-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/hdfsproxy/hdfsproxy-2.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/index/hadoop-index-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/streaming/hadoop-streaming-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/vaidya/hadoop-vaidya-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-ant-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-client-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-core-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-examples-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-minicluster-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-test-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-tools-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/ivy/ivy-2.1.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/contrib/thriftfs/lib/hadoopthriftapi.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/contrib/thriftfs/lib/libthrift.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/ftplet-api-1.0.0-SNAPSHOT.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/ftpserver-core-1.0.0-SNAPSHOT.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/ftpserver-server-1.0.0-SNAPSHOT.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/mina-core-2.0.0-M2-20080407.124109-12.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/org/apache/hadoop/mapred/test.jar

BUILD SUCCESSFUL
Total time: 2 seconds
[jifeng@jifeng04 HDFS_C_API]$ vi /home/jifeng/.bash_profile

[jifeng@jifeng04 HDFS_C_API]$ cat /home/jifeng/.bash_profile
# .bash_profile

# Get the aliases and functions
if [ -f ~/.bashrc ]; then
        . ~/.bashrc
fi

# User specific environment and startup programs

PATH=$PATH:$HOME/bin

export PATH
export JAVA_HOME=$HOME/jdk1.7.0_45
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_HOME=$HOME/hadoop/hadoop-1.2.1  
export ANT_HOME=$HOME/apache-ant-1.9.4  
  
export PATH=$PATH:$ANT_HOME/bin 
export CLASSPATH=$CLASSPATH:/home/jifeng/hadoop/hadoop-1.2.1/lib/asm-3.2.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/aspectjrt-1.6.11.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/aspectjtools-1.6.11.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-beanutils-1.7.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-beanutils-core-1.8.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-cli-1.2.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-codec-1.4.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-collections-3.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-configuration-1.6.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-daemon-1.0.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-digester-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-el-1.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-httpclient-3.0.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-io-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-lang-2.4.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-logging-1.1.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-logging-api-1.0.4.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-math-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/commons-net-3.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/core-3.1.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hadoop-capacity-scheduler-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hadoop-fairscheduler-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hadoop-thriftfs-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/hsqldb-1.8.0.10.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jackson-core-asl-1.8.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jackson-mapper-asl-1.8.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jasper-compiler-5.5.12.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jasper-runtime-5.5.12.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jdeb-0.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jersey-core-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jersey-json-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jersey-server-1.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jets3t-0.6.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jetty-6.1.26.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jetty-util-6.1.26.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jsch-0.1.42.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jsp-2.1/jsp-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/jsp-2.1/jsp-api-2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/junit-4.5.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/kfs-0.2.2.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/log4j-1.2.15.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/mockito-all-1.8.5.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/oro-2.0.8.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/servlet-api-2.5-20081211.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/slf4j-api-1.4.3.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/slf4j-log4j12-1.4.3.jar:/home/jifeng/hadoop/hadoop-1.2.1/lib/xmlenc-0.52.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/datajoin/hadoop-datajoin-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/failmon/hadoop-failmon-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/gridmix/hadoop-gridmix-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/hdfsproxy/hdfsproxy-2.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/index/hadoop-index-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/streaming/hadoop-streaming-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/contrib/vaidya/hadoop-vaidya-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-ant-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-client-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-core-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-examples-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-minicluster-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-test-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/hadoop-tools-1.2.1.jar:/home/jifeng/hadoop/hadoop-1.2.1/ivy/ivy-2.1.0.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/contrib/thriftfs/lib/hadoopthriftapi.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/contrib/thriftfs/lib/libthrift.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/ftplet-api-1.0.0-SNAPSHOT.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/ftpserver-core-1.0.0-SNAPSHOT.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/ftpserver-server-1.0.0-SNAPSHOT.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/lib/mina-core-2.0.0-M2-20080407.124109-12.jar:/home/jifeng/hadoop/hadoop-1.2.1/src/test/org/apache/hadoop/mapred/test.jar 

[jifeng@jifeng04 HDFS_C_API]$ source /home/jifeng/.bash_profile
[jifeng@jifeng04 HDFS_C_API]$ LD_LIBRARY_PATH=$HADOOP_HOME/c++/Linux-amd64-64/lib:$JAVA_HOME/jre/lib/amd64/server ./hdfs_cpp_demo
Welcome to HDFS C API!!!!
[jifeng@jifeng04 HDFS_C_API]$