java api 讀取hadoop中hdfs檔案系統內的檔案
阿新 • • 發佈:2019-02-04
hadoop與hdfs需要自己安裝,確保檔案存在於hdfs
只有一個main方法
Test.java
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException ;
import java.io.OutputStream;
public class Test {
public static void main(String[] args) throws IOException {
//建立configuration物件
Configuration conf = new Configuration();
//配置在etc/hadoop/core-site.xml fs.defaultFS
conf.set("fs.defaultFS","hdfs://192.168.163.132:8020" );
//建立FileSystem物件
//檢視hdfs叢集伺服器/user/passwd.txt的內容
FileSystem fs = FileSystem.get(conf);
FSDataInputStream is = fs.open(new Path("hdfs://192.168.163.132:8020/user/passwd.txt"));
File file = new File("a.txt");
System.out.println(file.getAbsolutePath());
OutputStream os = new FileOutputStream(file);
byte[] buff = new byte[1024];
int length = 0;
while ((length = is.read(buff)) != -1) {
System.out.println(new String(buff, 0, length));
os.write(buff, 0, length);
os.flush();
}
System.out.println(fs.getClass().getName());
}
}