ASP.NET实现Hadoop增删改查的示例代码

<?xml version="1.0" encoding="utf-8"?> <packages> <package version="4.0.20505.0" targetFramework="net46" /> <package version="5.2.0" targetFramework="net46" /> <package version="5.2.0" targetFramework="net46" /> <package version="0.12.5126.42915" targetFramework="net46" /> <package version="2.0.20505.0" targetFramework="net46" /> <package version="1.8.0.0" targetFramework="net46" /> <package version="4.5.11" targetFramework="net46" /> <package version="5.2.0" targetFramework="net46" /> <package version="2.0.4.1" targetFramework="net46" /> </packages>

HDFSAccess.cs

using Microsoft.Hadoop.WebHDFS; using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; namespace Physical { public sealed class HDFSAccess { private readonly WebHDFSClient webHDFSClient; public HDFSAccess(string uriString, string userName) { this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName); } public List<string> GetDirectories(string path) { var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result; return directoryStatus.Directories.Select(d => d.PathSuffix).ToList(); } public List<string> GetFiles(string path) { var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result; return directoryStatus.Files.Select(d => d.PathSuffix).ToList(); } public bool CreateDirectory(string path) { // 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.CreateDirectory(path).Result; } public bool DeleteDirectory(string path) { // 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.DeleteDirectory(path).Result; } public string CreateFile(string localFile, string remotePath) { // 传入远端路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.CreateFile(localFile, remotePath).Result; } public bool DeleteFile(string path) { // 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.DeleteDirectory(path).Result; } public HttpResponseMessage OpenFile(string path) { // 传入路径不包含根目录时，预设会在根目录「/」底下 return this.webHDFSClient.OpenFile(path).Result; } } }

Program.cs

using Physical; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; namespace MyTest { class Program { // HDFS cluster 客户端进入端点设定于 主机上 // 预设端点：[主机名称]:50070 // 预设帐号：Wu private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu"); static void Main(string[] args) { GetDirectoriesTest(); Console.WriteLine("----------------------------------------------"); GetFilesTest(); Console.WriteLine("----------------------------------------------"); DirectoryTest(); Console.WriteLine("----------------------------------------------"); FileTest(); Console.WriteLine("----------------------------------------------"); OpenFileTest(); Console.ReadKey(); } public void TestCleanup() { //取得根目录资料夹 var directories = access.GetDirectories(@"https://www.jb51.net/"); // 移除预设目录：tmp、user 外的目录 foreach (var directory in directories) { if ("tmp".Equals(directory) || "user".Equals(directory)) { continue; } else { access.DeleteDirectory(directory); } } // 取得根目录档案 var files = access.GetFiles(@"https://www.jb51.net/"); // 移除所有档案 foreach (var file in files) { access.DeleteFile(file); } // 移除 OpenFile 转存档案 File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg")); } //_传入根目录_预期回传预设目录 public static void GetDirectoriesTest() { // 预设根目录下有两个目录：tmp、user var expected = new List<string>() { "tmp", "user", }; var actual = access.GetDirectories(@"https://www.jb51.net/"); foreach (var item in actual) { Console.WriteLine(item); } } //_传入根目录_预期回传空集合 public static void GetFilesTest() { // 预设根目录下没有档案 var expected = new List<string>(); var actual = access.GetFiles(@"https://www.jb51.net/"); foreach (var item in actual) { Console.WriteLine(item); } } //_建立zzz目录_预期成功_预期根目录下有zzz目录_删除zzz目录_预期成功_预期根目录下无zzz目录 public static void DirectoryTest() { var directoryName = "zzz"; // 建立zzz目录 var boolCreateDirectory = access.CreateDirectory(directoryName); Console.WriteLine("建立zzz目录_预期成功:"+boolCreateDirectory); // 建立zzz目录_预期成功_预期根目录下有zzz目录 // 预设根目录下有三个目录：tmp、user、zzz var expectedCreateDirectory = new List<string>() { "tmp", "user", directoryName, }; var actualCreateDirectory = access.GetDirectories(@"https://www.jb51.net/"); foreach (var item in actualCreateDirectory) { Console.WriteLine(item); } Console.WriteLine("********************************************"); // 删除zzz目录 var boolDeleteDirectory = access.DeleteDirectory(directoryName); Console.WriteLine("删除zzz目录_预期成功:" + boolDeleteDirectory); // 删除zzz目录_预期成功_预期根目录下无zzz目录 // 预设根目录下有两个目录：tmp、user var expectedDeleteDirectory = new List<string>() { "tmp", "user", }; var actualDeleteDirectory = access.GetDirectories(@"https://www.jb51.net/"); foreach (var item in actualDeleteDirectory) { Console.WriteLine(item); } } //_建立Test档案_预期根目录下有Test档案_删除Test档案_预期成功_预期根目录下无Test档案 public static void FileTest() { var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg"); var remotePath = "Test.jpg"; // 建立Test档案 var boolCreateFile = access.CreateFile(localFile, remotePath); // 建立Test档案_预期根目录下有Test档案 var expectedCreateFile = new List<string>() { remotePath, }; var actualCreateFile = access.GetFiles(@"https://www.jb51.net/"); foreach (var item in actualCreateFile) { Console.WriteLine(item); } Console.WriteLine("********************************************"); // 删除Test档案 var boolDeleteFile = access.DeleteDirectory(remotePath); Console.WriteLine("删除Test档案_预期成功:"+boolDeleteFile); // 删除Test档案_预期成功_预期根目录下无Test档案 var expectedDeleteFile = new List<string>(); var actualDeleteFile = access.GetFiles(@"https://www.jb51.net/"); foreach (var item in actualDeleteFile) { Console.WriteLine(item); } } //_建立Test档案_预期根目录下有Test档案_取得Test档案_预期成功_预期回传Test档案Stream并转存成功 public static void OpenFileTest() { var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg"); var remotePath = "Test.jpg"; var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"); Console.WriteLine("saveFile:" + saveFile); Console.WriteLine("********************************************"); // 建立Test档案 var boolCreateFile = access.CreateFile(localFile, remotePath); // 建立Test档案_预期根目录下有Test档案 var expectedCreateFile = new List<string>() { remotePath, }; var actualCreateFile = access.GetFiles(@"https://www.jb51.net/"); foreach (var item in actualCreateFile) { Console.WriteLine(item); } Console.WriteLine("********************************************"); // 取得Test档案 var response = access.OpenFile(remotePath); // 取得Test档案_预期成功 response.EnsureSuccessStatusCode(); // 取得Test档案_预期成功_预期回传Test档案Stream并转存成功 using (var fs = File.Create(saveFile)) { response.Content.CopyToAsync(fs).Wait(); } Console.WriteLine(File.Exists(saveFile)); } } }

运行结果如图: