hadoop學習之路(2)
阿新 • • 發佈：2020-09-01
1.本地安裝hadoop(不安裝本地hadoop會報錯,雖然並不影響遠端的環境,但會報錯:Failed to locate the winutils binary in the hadoop binary path)
2.啟動hadoop環境,dfs,yarn,然後測試程式碼(DataNode埠與linux設定埠一致)
package org.example; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException;View Codeimport java.net.URI; import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils;import org.junit.Test; public class HDFSIO { // 把本地d盤上的zhang.txt檔案上傳到HDFS根目錄 @Test public void putFileToHDFS() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(newURI("hdfs://hadoop001:8020"), conf , "root"); // 2 獲取輸入流 FileInputStream fis = new FileInputStream(new File("d:/zhang.txt")); // 3 獲取輸出流 FSDataOutputStream fos = fs.create(new Path("/zhang.txt")); // 4 流的對拷 IOUtils.copyBytes(fis, fos, conf); // 5 關閉資源 IOUtils.closeStream(fos); IOUtils.closeStream(fis); fs.close(); } // 從HDFS上下載zhang.txt檔案到本地e盤上 @Test public void getFileFromHDFS() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 獲取輸入流 FSDataInputStream fis = fs.open(new Path("/san.txt")); // 3 獲取輸出流 FileOutputStream fos = new FileOutputStream(new File("d:/san.txt")); // 4 流的對拷 IOUtils.copyBytes(fis, fos, conf); // 5 關閉資源 IOUtils.closeStream(fos); IOUtils.closeStream(fis); fs.close(); } // 下載第一塊 @Test public void readFileSeek1() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 獲取輸入流 FSDataInputStream fis = fs.open(new Path("/hadoop-2.7.2.tar.gz")); // 3 獲取輸出流 FileOutputStream fos = new FileOutputStream(new File("d:/hadoop-2.7.2.tar.gz.part1")); // 4 流的對拷（只拷貝128m） byte[] buf = new byte[1024]; for (int i = 0; i < 1024 * 128; i++) { fis.read(buf); fos.write(buf); } // 5 關閉資源 IOUtils.closeStream(fos); IOUtils.closeStream(fis); fs.close(); } // 下載第二塊 @SuppressWarnings("resource") @Test public void readFileSeek2() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 獲取輸入流 FSDataInputStream fis = fs.open(new Path("/hadoop-2.7.2.tar.gz")); // 3 設定指定讀取的起點 fis.seek(1024*1024*128); // 4 獲取輸出流 FileOutputStream fos = new FileOutputStream(new File("d:/hadoop-2.7.2.tar.gz.part2")); // 5 流的對拷 IOUtils.copyBytes(fis, fos, conf); // 6 關閉資源 IOUtils.closeStream(fos); IOUtils.closeStream(fis); fs.close(); } }
package org.example; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.junit.Test; public class HDFSClient { public static void main(String[] args) throws IOException, Exception, URISyntaxException { Configuration conf = new Configuration(); // conf.set("fs.defaultFS", "hdfs://hadoop001:8020"); // 1 獲取hdfs客戶端物件 // FileSystem fs = FileSystem.get(conf ); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf, "root"); // 2 在hdfs上建立路徑 fs.mkdirs(new Path("/0529/dashen/zhang")); // 3 關閉資源 fs.close(); System.out.println("over"); } // 1 檔案上傳 @Test public void testCopyFromLocalFile() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取fs物件 Configuration conf = new Configuration(); conf.set("dfs.replication", "2"); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 執行上傳API fs.copyFromLocalFile(new Path("d:/zhang.txt"), new Path("/zhang.txt")); // 3 關閉資源 fs.close(); } // 2 檔案下載 @Test public void testCopyToLocalFile() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 執行下載操作 // fs.copyToLocalFile(new Path("/zhang.txt"), new Path("d:/zhang1.txt")); fs.copyToLocalFile(false, new Path("/zhang.txt"), new Path("d:/zhangzhang.txt"), true); // 3 關閉資源 fs.close(); } // 3 檔案刪除 @Test public void testDelete() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 檔案刪除 fs.delete(new Path("/0529"), true); // 3 關閉資源 fs.close(); } // 4 檔案更名 @Test public void testRename() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 執行更名操作 fs.rename(new Path("/zhang.txt"), new Path("/zhang1.txt")); // 3 關閉資源 fs.close(); } // 5 檔案詳情檢視 @Test public void testListFiles() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 檢視檔案詳情 RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true); while(listFiles.hasNext()){ LocatedFileStatus fileStatus = listFiles.next(); // 檢視檔名稱、許可權、長度、塊資訊 System.out.println(fileStatus.getPath().getName());// 檔名稱 System.out.println(fileStatus.getPermission());// 檔案許可權 System.out.println(fileStatus.getLen());// 檔案長度 BlockLocation[] blockLocations = fileStatus.getBlockLocations(); for (BlockLocation blockLocation : blockLocations) { String[] hosts = blockLocation.getHosts(); for (String host : hosts) { System.out.println(host); } } System.out.println("------test分割線--------"); } // 3 關閉資源 fs.close(); } // 6 判斷是檔案還是資料夾 @Test public void testListStatus() throws IOException, InterruptedException, URISyntaxException{ // 1 獲取物件 Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(new URI("hdfs://hadoop001:8020"), conf , "root"); // 2 判斷操作 FileStatus[] listStatus = fs.listStatus(new Path("/")); for (FileStatus fileStatus : listStatus) { if (fileStatus.isFile()) { // 檔案 System.out.println("f:"+fileStatus.getPath().getName()); }else{ // 資料夾 System.out.println("d:"+fileStatus.getPath().getName()); } } // 3 關閉資源 fs.close(); } }View Code