使用IDEA實現HDFS客戶端的簡單操作
阿新 • • 發佈：2018-12-19
新增依賴
<dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> <scope>test</scope> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.7.2</version> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.7.2</version> </dependency> <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.7.2</version> </dependency>
log4j.properties
log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n log4j.appender.logfile=org.apache.log4j.FileAppender log4j.appender.logfile.File=target/spring.log log4j.appender.logfile.layout=org.apache.log4j.PatternLayout log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n
hdfs-site.xml:優先級別:在linux裡面設定的相當於預設值 < 配置檔案 < 程式碼 <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="configuration.xsl"?> dfs.replication 1
package com.zyd; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.junit.Test; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; /** * Unit test for simple App. */ public class AppTest { @Test /** * HDFS獲取檔案系統 */ public void initHDFS() throws Exception{ // 1 建立配置資訊物件 Configuration configuration = new Configuration(); // 2 獲取檔案系統 FileSystem fs = FileSystem.get(configuration); // 3 列印檔案系統 System.out.println(fs.toString()); }
@Test
/**
* HDFS檔案上傳(測試引數優先順序)
*/
public void testCopyFromLocalFIle() throws URISyntaxException, IOException, InterruptedException {
//1 獲取檔案系統
Configuration configuration = new Configuration();
//configuration.set("dfs.replication","2");
FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
//2 上傳檔案
fs.copyFromLocalFile(new Path("D:/LICENSE.txt"),new Path("/idea.txt"));
//3 關閉資源
fs.close();
System.out.println("over");
}
/**
* HDFS檔案的下載
*/
@Test
public void testCopyToLocalFile() throws URISyntaxException, IOException, InterruptedException {
// 1 獲取檔案系統
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
// 2 執行下載操作
//boolean delSrc 指是否將原檔案刪除
//Path src 指要下載的檔案路徑
//Path dst 指將檔案下載到的路徑
//boolean useRawLocalFileSystem 是否開啟檔案校驗
fs.copyToLocalFile(false,new Path("/hello.txt"),new Path("e:/hadoop.txt"),true);
//3 關閉資源
fs.close();
System.out.println("over");
}
@Test
/**
* HDFS檔案目錄的建立
*/
public void testMkdir() throws URISyntaxException, IOException, InterruptedException {
//1.獲取檔案系統
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
//2.建立目錄
fs.mkdirs(new Path("/0906/zhangsan"));
//3.關閉資源
fs.close();
}
/**
* HDFS檔案的刪除
* @throws IOException
*/
@Test
public void testDelete() throws IOException, URISyntaxException, InterruptedException {
//1.獲取檔案系統
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
//2.執行刪除
fs.delete(new Path("/ok.txt"),true);
//3.關閉資源
fs.close();
}
/**
* HDFS 檔名的修改
* @throws URISyntaxException
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testRename() throws URISyntaxException, IOException, InterruptedException {
//1 獲取檔案系統
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
//2 修改檔名稱
fs.rename(new Path("/hello11.txt"),new Path("/Im.txt"));
//3. 關閉資源
fs.close();
}
@Test
public void testListFiles() throws URISyntaxException, IOException, InterruptedException {
//1 獲取檔案系統
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
//2 獲取檔案詳情
/**
* 思考:為什麼返回迭代器不是list之類的容器
* 集合佔記憶體多 一次全部拿取
* 迭代器一次拿一個
*/
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"),true);
while (listFiles.hasNext()){
LocatedFileStatus status = listFiles.next();
//輸出詳情
//檔名稱
System.out.println("檔名:"+status.getPath().getName());
//長度
System.out.println("檔案大小"+status.getLen());
//獲取組
System.out.println("所屬組:"+status.getGroup());
//許可權
System.out.println("許可權:"+status.getPermission());
//獲取儲存的塊資訊
BlockLocation[] blockLocations = status.getBlockLocations();
for (BlockLocation blockLocation:blockLocations){
//獲取塊的儲存的主機節點
String[] hosts = blockLocation.getHosts();
for (String host:hosts){
System.out.println("獲取塊的儲存的主機節點:"+host);
}
}
System.out.println("-------------------------");
}
fs.close();
}
列印結果:
檔名:ok.txt
檔案大小3778
所屬組:supergroup
許可權:rw-r--r--
獲取塊的儲存的主機節點:testnote01
獲取塊的儲存的主機節點:testnote02
獲取塊的儲存的主機節點:testnote03
-------------------------
檔名:a.txt
檔案大小6
所屬組:supergroup
許可權:rw-rw-rw-
獲取塊的儲存的主機節點:testnote03
獲取塊的儲存的主機節點:testnote02
-------------------------
/**
* HDFS檔案和資料夾的判斷
*/
@Test
public void testListStatus() throws URISyntaxException, IOException, InterruptedException {
//1 獲取檔案配置資訊
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://testnote01:9000"),configuration,"root");
//2 判斷是檔案還是資料夾
FileStatus[] listStatus = fs.listStatus(new Path("/"));
for (FileStatus fileStatus:listStatus){
//如果是檔案
if (fileStatus.isFile()){
System.out.println("檔案:"+fileStatus.getPath().getName());
}else {
System.out.println("資料夾:"+fileStatus.getPath().getName());
}
}
//關閉資源
fs.close();
}
}