客戶端對HDFS的操作
阿新 • • 發佈:2019-02-10
package cn.itcast.bigdata.hdfs; import java.net.URI; import java.util.Iterator; import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.junit.Before; import org.junit.Test; /** * * 客戶端去操作hdfs時,是有一個使用者身份的 * 預設情況下,hdfs客戶端api會從jvm中獲取一個引數來作為自己的使用者身份:-DHADOOP_USER_NAME=hadoop * * 也可以在構造客戶端fs物件時,通過引數傳遞進去 * @author * */ public class HdfsClientDemo { FileSystem fs = null; Configuration conf = null; @Before public void init() throws Exception{ conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://master:9000"); //拿到一個檔案系統操作的客戶端例項物件 /*fs = FileSystem.get(conf);*/ //可以直接傳入 uri和使用者身份 fs = FileSystem.get(new URI("hdfs://master:9000"),conf,"hadoop"); //最後一個引數為使用者名稱 } @Test public void testUpload() throws Exception { Thread.sleep(2000); fs.copyFromLocalFile(new Path("G:/access.log"), new Path("/access.log.copy")); fs.close(); } @Test public void testDownload() throws Exception { fs.copyToLocalFile(new Path("/access.log.copy"), new Path("d:/")); fs.close(); } @Test public void testConf(){ Iterator<Entry<String, String>> iterator = conf.iterator(); while (iterator.hasNext()) { Entry<String, String> entry = iterator.next(); System.out.println(entry.getValue() + "--" + entry.getValue());//conf載入的內容 } } /** * 建立目錄 */ @Test public void makdirTest() throws Exception { boolean mkdirs = fs.mkdirs(new Path("/aaa/bbb")); System.out.println(mkdirs); } /** * 刪除 */ @Test public void deleteTest() throws Exception{ boolean delete = fs.delete(new Path("/aaa"), true);//true, 遞迴刪除 System.out.println(delete); } @Test public void listTest() throws Exception{ FileStatus[] listStatus = fs.listStatus(new Path("/")); for (FileStatus fileStatus : listStatus) { System.err.println(fileStatus.getPath()+"================="+fileStatus.toString()); } //會遞迴找到所有的檔案 RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true); while(listFiles.hasNext()){ LocatedFileStatus next = listFiles.next(); String name = next.getPath().getName(); Path path = next.getPath(); System.out.println(name + "---" + path.toString()); } } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://master:9000"); //拿到一個檔案系統操作的客戶端例項物件 FileSystem fs = FileSystem.get(conf); fs.copyFromLocalFile(new Path("G:/access.log"), new Path("/access.log.copy")); fs.close(); } }