大数据原理及应用实验

1.hadoop安装

http://dblab.xmu.edu.cn/blog/install-hadoop/

2.eclipse安装

https://mirrors.tuna.tsinghua.edu.cn/eclipse/technology/epp/downloads/release/2018-09/R/

sudo tar -zxvf ~/下载/eclipse-jee-2018-09-linux-gtk-x86_64.tar.gz

 sudo mv eclipse /usr/local/

sudo ln -s /usr/lib/jvm/default-java/jre jre

cd /usr/share/applications

sudo vim eclipse.desktop

[Desktop Entry]
Encoding=UTF-8
Name=Eclipse
Comment=Eclipse
Exec=/usr/local/eclipse/eclipse
Icon=/usr/local/eclipse/icon.xpm
Terminal=false
StartupNotify=true
Type=Application
Categories=Application;Development;

 

sudo chmod u+x /usr/share/applications/eclipse.desktop

3.HDBS实验

https://www.cnblogs.com/MessiXiaoMo3334/p/12643994.html

https://blog.csdn.net/nenguou04/article/details/88878938

https://blog.csdn.net/Fire_to_cheat_/article/details/80856850

importorg.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import java.io.*;



public class MyFSDataInputStream extendsFSDataInputStream {

 public MyFSDataInputStream(InputStream in) {

   super(in);

  }



  publicstatic String readline(BufferedReader br) throws IOException {



   char[] data = new char[1024];

   int read = -1;

   int off = 0;

   while ((read = br.read(data, off, 1)) != -1) {

     if (String.valueOf(data[off]).equals("\n")) {

       off += 1;

       break;

     }

     off += 1;

    }

   if (off > 0) {

     return String.valueOf(data);

    }else {

     return null;

    }

  }



  /**

   *读取文件内容

   */

 public static void cat(Configuration conf, String remoteFilePath) throwsIOException {

   FileSystem fs = FileSystem.get(conf);

   Path remotePath = new Path(remoteFilePath);

   FSDataInputStream in = fs.open(remotePath);

   BufferedReader br = new BufferedReader(new InputStreamReader(in));

   String line = null;

   while ((line = MyFSDataInputStream.readline(br)) != null) {

     System.out.println(line);

    }

   br.close();

   in.close();

   fs.close();

  }





 public static void main(String[] args) {

   Configuration conf = new Configuration();

   conf.set("fs.default.name","hdfs://localhost:9000");

   String remoteFilePath = "/user/hadoop/input/local.txt"; //HDFS路径



   try {

     MyFSDataInputStream.cat(conf, remoteFilePath);

    }catch (Exception e) {

     e.printStackTrace();

    }

  }

}

import java.io.IOException;

import java.io.InputStream;

import java.net.MalformedURLException;

import java.net.URL;



importorg.apache.hadoop.fs.FsUrlStreamHandlerFactory;

import org.apache.hadoop.io.IOUtils;



public class ShowTheContent {

    static{

       URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());

    }



   public static void cat(String remoteFilePath){

       try {

           InputStream in=newURL("hdfs","localhost",9000,remoteFilePath).openStream();

           IOUtils.copyBytes(in,System.out, 4096,false);

           IOUtils.closeStream(in);



       } catch ( IOException e) {

           // TODO Auto-generated catch block

           e.printStackTrace();

       }

    }

   public static void main(String[] args){

       String remoteFilePath="/user/hadoop/input/local.txt";//fileroute

       System.out.println("read file:"+remoteFilePath);

       ShowTheContent.cat(remoteFilePath);

       System.out.println("\n read finish!");

    }

}

 

4.Hbase实验

下载

https://mirrors.tuna.tsinghua.edu.cn/apache/hbase/

安装

http://dblab.xmu.edu.cn/blog/install-hbase/#more-588

实验代码:

https://blog.csdn.net/AthlenaA/article/details/87909768

https://www.freesion.com/article/7298491561/

 

5.Mapreduce实验

https://www.jianshu.com/p/7328bb45a7cd