Java操作hdfs的檔案增刪改查
阿新 • • 發佈:2019-02-02
package cn.itcast.hadoop.hdfs; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.junit.Before; import org.junit.Test; public class HdfsUtil { FileSystem fs = null; @Before public void init() throws Exception{ //讀取classpath下的xxx-site.xml 配置檔案,並解析其內容,封裝到conf物件中 Configuration conf = new Configuration(); //也可以在程式碼中對conf中的配置資訊進行手動設定,會覆蓋掉配置檔案中的讀取的值 conf.set("fs.defaultFS", "hdfs://weekend110:9000/"); //根據配置資訊,去獲取一個具體檔案系統的客戶端操作例項物件 fs = FileSystem.get(new URI("hdfs://weekend110:9000/"),conf,"hadoop"); } /** * 上傳檔案,比較底層的寫法 * * @throws Exception */ @Test public void upload() throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://weekend110:9000/"); FileSystem fs = FileSystem.get(conf); Path dst = new Path("hdfs://weekend110:9000/aa/qingshu.txt"); FSDataOutputStream os = fs.create(dst); FileInputStream is = new FileInputStream("c:/qingshu.txt"); IOUtils.copy(is, os); } /** * 上傳檔案,封裝好的寫法 * @throws Exception * @throws IOException */ @Test public void upload2() throws Exception, IOException{ fs.copyFromLocalFile(new Path("c:/qingshu.txt"), new Path("hdfs://weekend110:9000/aaa/bbb/ccc/qingshu2.txt")); } /** * 下載檔案 * @throws Exception * @throws IllegalArgumentException */ @Test public void download() throws Exception { fs.copyToLocalFile(new Path("hdfs://weekend110:9000/aa/qingshu2.txt"), new Path("c:/qingshu2.txt")); } /** * 檢視檔案資訊 * @throws IOException * @throws IllegalArgumentException * @throws FileNotFoundException * */ @Test public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException { // listFiles列出的是檔案資訊,而且提供遞迴遍歷 RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true); while(files.hasNext()){ LocatedFileStatus file = files.next(); Path filePath = file.getPath(); String fileName = filePath.getName(); System.out.println(fileName); } System.out.println("---------------------------------"); //listStatus 可以列出檔案和資料夾的資訊,但是不提供自帶的遞迴遍歷 FileStatus[] listStatus = fs.listStatus(new Path("/")); for(FileStatus status: listStatus){ String name = status.getPath().getName(); System.out.println(name + (status.isDirectory()?" is dir":" is file")); } } /** * 建立資料夾 * @throws Exception * @throws IllegalArgumentException */ @Test public void mkdir() throws IllegalArgumentException, Exception { fs.mkdirs(new Path("/aaa/bbb/ccc")); } /** * 刪除檔案或資料夾 * @throws IOException * @throws IllegalArgumentException */ @Test public void rm() throws IllegalArgumentException, IOException { fs.delete(new Path("/aa"), true); } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://weekend110:9000/"); FileSystem fs = FileSystem.get(conf); FSDataInputStream is = fs.open(new Path("/jdk-7u65-linux-i586.tar.gz")); FileOutputStream os = new FileOutputStream("c:/jdk7.tgz"); IOUtils.copy(is, os); } }