hama(0.7.1)编译运行PiEstimator.java文件

hama(0.7.1)下载

百度网盘hama(0.7.1)
提取码：a765

前提条件

hama安装参考: 林子雨大数据课程Hama单机环境安装配置
需将hama的bin目录添加到环境变量PATH中.

classpath

执行过程中发现classpath添加目录下所有jar文件需将文件名全部写上，只有目录不行。(原因暂不清楚)

PiEstimator.java

import java.io.IOException;  
 
import org.apache.commons.logging.Log;  
import org.apache.commons.logging.LogFactory;  
import org.apache.hadoop.fs.FSDataInputStream;  
import org.apache.hadoop.fs.FileStatus;  
import org.apache.hadoop.fs.FileSystem;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.DoubleWritable;  
import org.apache.hadoop.io.IOUtils;  
import org.apache.hadoop.io.NullWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hama.HamaConfiguration;  
import org.apache.hama.bsp.BSP;  
import org.apache.hama.bsp.BSPJob;  
import org.apache.hama.bsp.BSPJobClient;  
import org.apache.hama.bsp.BSPPeer;  
import org.apache.hama.bsp.ClusterStatus;  
import org.apache.hama.bsp.FileOutputFormat;  
import org.apache.hama.bsp.NullInputFormat;  
import org.apache.hama.bsp.TextOutputFormat;  
import org.apache.hama.bsp.sync.SyncException;  
 
public class PiEstimator  
{  
    private static Path TMP_OUTPUT = new Path("/tmp/pi-"  
        + System.currentTimeMillis());  
 
    public static class MyEstimator  
        extends  
        BSP<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable>  
    {  
    public static final Log LOG = LogFactory.getLog(MyEstimator.class);  
    private String masterTask;  
    private static final int iterations = 10000;  
 
    @Override  
    public void bsp(  
        BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)  
        throws IOException, SyncException, InterruptedException  
    {  
 
        int in = 0;  
        for (int i = 0; i < iterations; i++)  
        {  
        double x = 2.0 * Math.random() - 1.0, y = 2.0 * Math.random() - 1.0;  
        if ((Math.sqrt(x * x + y * y) < 1.0))  
        {  
            in++;  
        }  
        }  
 
        double data = 4.0 * in / iterations;  
 
        peer.send(masterTask, new DoubleWritable(data));  
        peer.sync();  
    }  
 
    @Override  
    public void setup(  
        BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)  
        throws IOException  
    {  
        // Choose one as a master  
        this.masterTask = peer.getPeerName(peer.getNumPeers() / 2);  
    }  
 
    @Override  
    public void cleanup(  
        BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)  
        throws IOException  
    {  
        if (peer.getPeerName().equals(masterTask))  
        {  
        double pi = 0.0;  
        int numPeers = peer.getNumCurrentMessages();  
        DoubleWritable received;  
        while ((received = peer.getCurrentMessage()) != null)  
        {  
            pi += received.get();  
        }  
 
        pi = pi / numPeers;  
        peer.write(new Text("Estimated value of PI is"),  
            new DoubleWritable(pi));  
        }  
    }  
    }  
 
    static void printOutput(HamaConfiguration conf) throws IOException  
    {  
    FileSystem fs = FileSystem.get(conf);  
    FileStatus[] files = fs.listStatus(TMP_OUTPUT);  
    for (int i = 0; i < files.length; i++)  
    {  
        if (files[i].getLen() > 0)  
        {  
        FSDataInputStream in = fs.open(files[i].getPath());  
        IOUtils.copyBytes(in, System.out, conf, false);  
        in.close();  
        break;  
        }  
    }  
 
    fs.delete(TMP_OUTPUT, true);  
    }  
 
    public static void main(String[] args) throws InterruptedException,  
        IOException, ClassNotFoundException  
    {  
    // BSP job configuration  
    HamaConfiguration conf = new HamaConfiguration();  
 
    BSPJob bsp = new BSPJob(conf, PiEstimator.class);  
    // Set the job name  
    bsp.setJobName("Pi Estimation Example");  
    bsp.setBspClass(MyEstimator.class);  
    bsp.setInputFormat(NullInputFormat.class);  
    bsp.setOutputKeyClass(Text.class);  
    bsp.setOutputValueClass(DoubleWritable.class);  
    bsp.setOutputFormat(TextOutputFormat.class);  
    FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);  
 
    BSPJobClient jobClient = new BSPJobClient(conf);  
    ClusterStatus cluster = jobClient.getClusterStatus(true);  
 
    if (args.length > 0)  
    {  
        bsp.setNumBspTask(Integer.parseInt(args[0]));  
    } else  
    {  
        // Set to maximum  
        bsp.setNumBspTask(cluster.getMaxTasks());  
    }  
 
    long startTime = System.currentTimeMillis();  
    if (bsp.waitForCompletion(true))  
    {  
        printOutput(conf);  
        System.out.println("Job Finished in "  
            + (System.currentTimeMillis() - startTime) / 1000.0  
            + " seconds");  
    }  
    }  
}  

runhama.sh

#!/bin/bash
for jar in /usr/local/hama/lib/*.jar 
do
    export classpath=$classpath:$jar
done

for jar in /usr/local/hama/*.jar 
do
    export classpath=$classpath:$jar
done

javac -classpath $classpath PiEstimator.java
jar -cvf PiEstimator.jar *.class
hama jar PiEstimator.jar PiEstimator

命令行执行

source   runhama.sh

结果

added manifest
adding: PiEstimator.class(in = 3422) (out= 1705)(deflated 50%)
adding: PiEstimator$MyEstimator.class(in = 2441) (out= 1178)(deflated 51%)
19/10/23 22:07:17 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
19/10/23 22:07:17 INFO Configuration.deprecation: user.name is deprecated. Instead, use mapreduce.job.user.name
19/10/23 22:07:17 WARN conf.Configuration: org.apache.hadoop.fs.ChecksumFileSystem$FSDataBoundedInputStream@48fa0f47:an attempt to override final parameter: mapreduce.job.end-notification.max.retry.interval;  Ignoring.
19/10/23 22:07:17 WARN conf.Configuration: org.apache.hadoop.fs.ChecksumFileSystem$FSDataBoundedInputStream@48fa0f47:an attempt to override final parameter: mapreduce.job.end-notification.max.attempts;  Ignoring.
19/10/23 22:07:17 INFO Configuration.deprecation: user.name is deprecated. Instead, use mapreduce.job.user.name
19/10/23 22:07:18 INFO bsp.LocalBSPRunner: Setting up a new barrier for 10 tasks!
19/10/23 22:07:18 INFO bsp.BSPJobClient: Running job: job_localrunner_0001
19/10/23 22:07:18 INFO Configuration.deprecation: mapred.cache.localFiles is deprecated. Instead, use mapreduce.job.cache.local.files
19/10/23 22:07:21 INFO bsp.BSPJobClient: Current supersteps number: 0
19/10/23 22:07:21 INFO bsp.BSPJobClient: The total number of supersteps: 0
19/10/23 22:07:21 INFO bsp.BSPJobClient: Counters: 7
19/10/23 22:07:21 INFO bsp.BSPJobClient:   org.apache.hama.bsp.BSPPeerImpl$PeerCounter
19/10/23 22:07:21 INFO bsp.BSPJobClient:     TIME_IN_SYNC_MS=112
19/10/23 22:07:21 INFO bsp.BSPJobClient:     TASK_OUTPUT_RECORDS=1
19/10/23 22:07:21 INFO bsp.BSPJobClient:     SUPERSTEP_SUM=10
19/10/23 22:07:21 INFO bsp.BSPJobClient:     TOTAL_MESSAGES_SENT=10
19/10/23 22:07:21 INFO bsp.BSPJobClient:     TOTAL_MESSAGES_RECEIVED=10
19/10/23 22:07:21 INFO bsp.BSPJobClient:   org.apache.hama.bsp.JobInProgress$JobCounter
19/10/23 22:07:21 INFO bsp.BSPJobClient:     LAUNCHED_TASKS=10
19/10/23 22:07:21 INFO bsp.BSPJobClient:     SUPERSTEPS=0
Estimated value of PI is	3.15016
Job Finished in 3.969 seconds