php连接webhdfs,ASP.NET实现Hadoop增删改查的示例代码

本文介绍了ASP.NET实现Hadoop增删改查的示例代码，分享给大家，具体如下：

packages.config

HDFSAccess.cs

using Microsoft.Hadoop.WebHDFS;

using System;

using System.Collections.Generic;

using System.Linq;

using System.Net.Http;

namespace Physical

{

public sealed class HDFSAccess

{

private readonly WebHDFSClient webHDFSClient;

public HDFSAccess(string uriString, string userName)

{

this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName);

}

public List GetDirectories(string path)

{

var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

return directoryStatus.Directories.Select(d => d.PathSuffix).ToList();

}

public List GetFiles(string path)

{

var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

return directoryStatus.Files.Select(d => d.PathSuffix).ToList();

}

public bool CreateDirectory(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.CreateDirectory(path).Result;

}

public bool DeleteDirectory(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.DeleteDirectory(path).Result;

}

public string CreateFile(string localFile, string remotePath)

{

// 传入远端路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.CreateFile(localFile, remotePath).Result;

}

public bool DeleteFile(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.DeleteDirectory(path).Result;

}

public HttpResponseMessage OpenFile(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.OpenFile(path).Result;

}

}

}

Program.cs

using Physical;

using System;

using System.Collections.Generic;

using System.IO;

using System.Linq;

using System.Text;

using System.Threading.Tasks;

namespace MyTest

{

class Program

{

// HDFS cluster 客户端进入端点设定于 主机上

// 预设端点：http://[主机名称]:50070

// 预设帐号：Wu

private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu");

static void Main(string[] args)

{

GetDirectoriesTest();

Console.WriteLine("----------------------------------------------");

GetFilesTest();

Console.WriteLine("----------------------------------------------");

DirectoryTest();

Console.WriteLine("----------------------------------------------");

FileTest();

Console.WriteLine("----------------------------------------------");

OpenFileTest();

Console.ReadKey();

}

public void TestCleanup()

{

//取得根目录资料夹

var directories = access.GetDirectories(@"/");

// 移除预设目录：tmp、user 外的目录

foreach (var directory in directories)

{

if ("tmp".Equals(directory) || "user".Equals(directory))

{

continue;

}

else

{

access.DeleteDirectory(directory);

}

}

// 取得根目录档案

var files = access.GetFiles(@"/");

// 移除所有档案

foreach (var file in files)

{

access.DeleteFile(file);

}

// 移除 OpenFile 转存档案

File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"));

}

//_传入根目录_预期回传预设目录

public static void GetDirectoriesTest()

{

// 预设根目录下有两个目录：tmp、user

var expected = new List() { "tmp", "user", };

var actual = access.GetDirectories(@"/");

foreach (var item in actual)

{

Console.WriteLine(item);

}

}

//_传入根目录_预期回传空集合

public static void GetFilesTest()

{

// 预设根目录下没有档案

var expected = new List();

var actual = access.GetFiles(@"/");

foreach (var item in actual)

{

Console.WriteLine(item);

}

}

//_建立zzz目录_预期成功_预期根目录下有zzz目录_删除zzz目录_预期成功_预期根目录下无zzz目录

public static void DirectoryTest()

{

var directoryName = "zzz";

// 建立zzz目录

var boolCreateDirectory = access.CreateDirectory(directoryName);

Console.WriteLine("建立zzz目录_预期成功:"+boolCreateDirectory);

// 建立zzz目录_预期成功_预期根目录下有zzz目录

// 预设根目录下有三个目录：tmp、user、zzz

var expectedCreateDirectory = new List() { "tmp", "user", directoryName, };

var actualCreateDirectory = access.GetDirectories(@"/");

foreach (var item in actualCreateDirectory)

{

Console.WriteLine(item);

}

Console.WriteLine("********************************************");

// 删除zzz目录

var boolDeleteDirectory = access.DeleteDirectory(directoryName);

Console.WriteLine("删除zzz目录_预期成功:" + boolDeleteDirectory);

// 删除zzz目录_预期成功_预期根目录下无zzz目录

// 预设根目录下有两个目录：tmp、user

var expectedDeleteDirectory = new List() { "tmp", "user", };

var actualDeleteDirectory = access.GetDirectories(@"/");

foreach (var item in actualDeleteDirectory)

{

Console.WriteLine(item);

}

}

//_建立Test档案_预期根目录下有Test档案_删除Test档案_预期成功_预期根目录下无Test档案

public static void FileTest()

{

var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");

var remotePath = "Test.jpg";

// 建立Test档案

var boolCreateFile = access.CreateFile(localFile, remotePath);

// 建立Test档案_预期根目录下有Test档案

var expectedCreateFile = new List() { remotePath, };

var actualCreateFile = access.GetFiles(@"/");

foreach (var item in actualCreateFile)

{

Console.WriteLine(item);

}

Console.WriteLine("********************************************");

// 删除Test档案

var boolDeleteFile = access.DeleteDirectory(remotePath);

Console.WriteLine("删除Test档案_预期成功:"+boolDeleteFile);

// 删除Test档案_预期成功_预期根目录下无Test档案

var expectedDeleteFile = new List();

var actualDeleteFile = access.GetFiles(@"/");

foreach (var item in actualDeleteFile)

{

Console.WriteLine(item);

}

}

//_建立Test档案_预期根目录下有Test档案_取得Test档案_预期成功_预期回传Test档案Stream并转存成功

public static void OpenFileTest()

{

var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");

var remotePath = "Test.jpg";

var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg");

Console.WriteLine("saveFile:" + saveFile);

Console.WriteLine("********************************************");

// 建立Test档案

var boolCreateFile = access.CreateFile(localFile, remotePath);

// 建立Test档案_预期根目录下有Test档案

var expectedCreateFile = new List() { remotePath, };

var actualCreateFile = access.GetFiles(@"/");

foreach (var item in actualCreateFile)

{

Console.WriteLine(item);

}

Console.WriteLine("********************************************");

// 取得Test档案

var response = access.OpenFile(remotePath);

// 取得Test档案_预期成功

response.EnsureSuccessStatusCode();

// 取得Test档案_预期成功_预期回传Test档案Stream并转存成功

using (var fs = File.Create(saveFile))

{

response.Content.CopyToAsync(fs).Wait();

}

Console.WriteLine(File.Exists(saveFile));

}

}

}

运行结果如图:

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持脚本之家。