HDFS基本操作的API
阿新 • • 發佈：2018-10-16
clas hello iterator 調用方法 fsd for 副本 == and
一、從hdfs下載文件到windows本地：
package com.css.hdfs01;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
* 從hdfs下載文件到windows本地
*
* 註意：
* 1.需要配置hadoop環境變量
* 2.需要編譯好的winutils包
二、hdfs常用的API：
package com.css.hdfs02;
import java.io.IOException;
三、hdfs讀寫文件：
package com.css.hdfs03;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
/**
* hdfs讀寫文件
*/
public class ReadData {
FileSystem fs =null;
@Before
public void init() throws IOException, InterruptedException, URISyntaxException {
// 1.加載配置
Configuration conf = new Configuration();
// 2.構造客戶端
fs = FileSystem.get(new URI("hdfs://192.168.146.132:9000/"), conf, "root");
}
/**
* 讀數據方式一
*/
@Test
public void testReadData1() throws IllegalArgumentException, IOException{
// 1.拿到流
FSDataInputStream in = fs.open(new Path("/a.txt"));
byte[] buf = new byte[1024];
in.read(buf);
System.out.println(new String(buf));
// 2.關閉資源
in.close();
fs.close();
}
/**
* 讀數據方式二
*/
@Test
public void testReadData2() throws IllegalArgumentException, IOException{
// 1.拿到流
FSDataInputStream in = fs.open(new Path("/hdfs-site.xml"));
// 2.緩沖流
BufferedReader br = new BufferedReader(new InputStreamReader(in, "UTF-8"));
// 3.按行讀取
String line = null;
// 4.讀數據
while ((line = br.readLine()) != null) {
System.out.println(line);
}
// 5.關閉資源
br.close();
in.close();
fs.close();
}
/**
* 讀取hdfs中指定偏移量
*/
@Test
public void testRandomRead() throws IllegalArgumentException, IOException{
// 1.拿到流
FSDataInputStream in = fs.open(new Path("/hdfs-site.xml"));
// 2.移動文件讀取指針到指定位置
in.seek(14);
byte[] b = new byte[5];
// 3.從指針位置開始讀取數組b的長度個字節
in.read(b);
System.out.println(new String(b));
// 4.關閉資源
in.close();
}
/**
* 在hdfs中寫數據方式一
*/
@Test
public void testWriteData() throws IllegalArgumentException, IOException{
// 1.輸出流
FSDataOutputStream out = fs.create(new Path("/windows.txt"), false);
// 2.輸入流
FileInputStream in = new FileInputStream("C:\\Users\\Administrator\\Desktop\\1012.txt");
byte[] buf = new byte[1024];
int read = 0;
while ((read = in.read(buf)) != -1) {
out.write(buf, 0, read);
}
// 3.關閉資源
in.close();
out.close();
fs.close();
}
/**
* 在hdfs中寫數據方式二
*/
@Test
public void testWriteData1() throws IllegalArgumentException, IOException{
// 1.創建輸出流
FSDataOutputStream out = fs.create(new Path("/love"));
// 2.寫數據
out.write("Areyouokmylove".getBytes());
// 3.關閉資源
IOUtils.closeStream(out);
fs.close();
}
}
HDFS基本操作的API