C# Hadoop学习笔记（九）—C#的分布式计算简单类图

一、创建Master节点

    1）负责集群整体连接认证（Connect方法）

    2）负责集群整体节点状态算法

代码如下：

[ServiceContract(ProtectionLevel = System.Net.Security.ProtectionLevel.None)]
    public interface IMasterTracker
    {
        /// <summary>
        /// Master节点注册。
        /// </summary>
        /// <param name="NodeInfo"></param>
        /// <returns></returns>
        [OperationContract]
        string RegisterJobNode(NodeInfo NodeInfo);

        /// <summary>
        /// 集群连接。
        /// </summary>
        /// <param name="NodeInfo"></param>
        /// <returns></returns>
        [OperationContract]
        FCloudConnectResult Connect(MasterConfigration masterConfig);

        /// <summary>
        /// 数据节点注册。
        /// </summary>
        /// <param name="NodeInfo"></param>
        /// <returns></returns>
        [OperationContract]
        string RegisterDataNode(NodeInfo NodeInfo);

        /// <summary>
        /// 计算节点信息清单.
        /// </summary>
        /// <returns></returns>
        [OperationContract]
        List<NodeInfo> GetDataNodeInfo();

        /// <summary>
        /// 活跃的计算节点信息清单.
        /// </summary>
        /// <returns></returns>
        [OperationContract]
        List<NodeInfo> GetValidDataNodes();

二、创建Job调度节点

       1）负责工作的调度

       2）分发任务给各个节点

       3）合并计算Combin

       4）接收节点计算结果反馈

       5）动态调整均衡

代码如下：

public interface IJobTracker
    {
        /// <summary>
        /// 操作契约（泛型），执行工作
        /// </summary>
        /// <typeparam name="TMapper"></typeparam>
        /// <typeparam name="TReducer"></typeparam>
        /// <typeparam name="TCombiner"></typeparam>
        /// <param name="config"></param>
        /// <returns></returns>
        [OperationContract]
        List<MapReduceResult> ExecuteGeneric<TMapper, TReducer, TCombiner>(JobConfiguration config)
            where TMapper : MapperBase, new()
            where TReducer : ReducerBase, new()
            where TCombiner : CombinerBase, new();
        /// <summary>
        /// 获取任务信息
        /// </summary>
        /// <returns></returns>
        [OperationContract]
        List<JobInfo> GetJobInfo();

        /// <summary>
        /// 更新任务状态
        /// </summary>
        /// <returns></returns>
        [OperationContract]
        void RefrashTaskGeneric<TCombiner>(TaskInfo task)
            where TCombiner : CombinerBase, new();

        /// <summary>
        /// 更新任务状态
        /// </summary>
        /// <returns></returns>
        [OperationContract]
        void RefrashTask(CombinerBase mCombinerBase,TaskInfo task);

        /// <summary>
        /// 操作契约，执行工作
        /// </summary>
        /// <param name="mMapperBase"></param>
        /// <param name="mReducerBase"></param>
        /// <param name="mCombinerBase"></param>
        /// <param name="config"></param>
        /// <returns></returns>
        [OperationContract]
        MapReduceResult Execute(MapperBase mMapperBase, ReducerBase mReducerBase, 
                         CombinerBase mCombinerBase, JobConfiguration config);
    }

三、创建任务节点

      1）负责心跳通知主节点

      2）负责执行工作节点分派的任务

      3）拆分Map，执行Reduce

      4）结果通知给Job节点

代码如下：

 public interface ITaskTracker
    {
        /// <summary>
        /// 操作契约（泛型），执行单节点计算任务
        /// </summary>
        /// <typeparam name="TMapper"></typeparam>
        /// <typeparam name="TReducer"></typeparam>
        /// <typeparam name="TCombiner"></typeparam>
        /// <param name="config"></param>
        /// <returns></returns>
        [OperationContract]
        List<MapReduceResult> ExecuteGeneric<TMapper, TReducer, TCombiner>(TaskConfigration config)
            where TMapper : MapperBase, new()
            where TReducer : ReducerBase, new()
            where TCombiner : CombinerBase, new();


        /// <summary>
        /// 获取任务信息
        /// </summary>
        /// <returns></returns>
        [OperationContract]
        List<TaskInfo> GetTaskInfo();


        /// <summary>
        /// 操作契约，执行单节点计算任务
        /// </summary>
        /// <param name="mapperBase"></param>
        /// <param name="reducerBase"></param>
        /// <param name="combinerBase"></param>
        /// <param name="config"></param>
        /// <returns></returns>
        [OperationContract]
        List<MapReduceResult> Execute(MapperBase mapperBase, ReducerBase reducerBase, CombinerBase combinerBase, TaskConfigration config);
    }

四、集群API设计

        1）负责存储的扩展，DEMO中仅支持文件存储，未使用HDFS

        2）负责串接所有通信API，包括连接集群、配置调度器、配置存储器、启动Job、开始一个工作

        3）返回结果提供给客户端

基本类图：

 五、客户端调用

（一）调用分布式计算接口：

 //初始化云计算环境
            Uri myUri = new Uri("net.tcp://127.0.0.1:55010/MasterTracker");
            //连接云环境（账号验证等方面没做处理）
            IFCloud myCluster = FCloud.Connect(myUri,string.Empty,string.Empty);
            //初始化任务信息，一般来说输入和输出是必须的
            JobConfiguration config = new JobConfiguration();
            //执行一条任务
            MapReduceResult jobResult = myCluster.MapReduceJob.Execute<MyMaper, MyReducer, MyCombiner>(config);
            //完成返回
            int exitCode = jobResult.Info.ExitCode;
            if (exitCode == 0)
            {
             //剩余的客户端处理。。。。。
            }

（二）调用分布式存储接口：

                          StorageConfiguration storageConfig = new StorageConfiguration();

                          //执行一条存储

                          StorageResult createResult = myCluster.StorageSystem.Write<MyInputer>(storageConfig);

（三）调用监控接口：

                 MonitorConfiguration  config = new MonitorConfiguration();

                 //集群运行时信息监控

                 myCluster.Monitor.OnRuntimeMonitor += new MonitorEquipmentHandler(test.DisplayRuntimeInfo);

                 //各节点运行信息，任务计算信息监控

                  myCluster.Monitor.OnDetailMonitor += new MonitorEquipmentHandler(test.DisplayDetailInfo);

                 //集群CPU等基本信息监控

                 myCluster.Monitor.OnBasicMonitor += new MonitorEquipmentHandler(test.DisplayBasicInfo);

                 //执行监控

                  result = myCluster.Monitor.ExecuteMonitor<MonitorDetailCallback>(config);