hdfs2.9.0簡單開發
阿新 • • 發佈:2018-11-11
1、maven工程的構建,hadoop用的版本是2.9.0。jdk的版本為1.8.0_131
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.9.0</version> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common --> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.9.0</version> <exclusions> <exclusion> <groupId>jdk.tools</groupId> <artifactId>jdk.tools</artifactId> </exclusion> </exclusions> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.9.0</version>
</dependency>
<dependency> <groupId>jdk.tools</groupId> <artifactId>jdk.tools</artifactId> <version>1.8.0_131</version> <scope>system</scope> <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath> </dependency>
2、對 hdfs的簡單操作
//檔案系統
FileSystem fileSystem;
//初始化檔案系統
public void init() throws Exception{
//讀取資料由平臺上的協議確定
URI uri = new URI("hdfs://dtbigdata:9000");
Configuration conf = new Configuration();
fileSystem = FileSystem.get(uri, conf);
}
//檢視目錄
public void Catalog() throws Exception{
Path path = new Path("/user");
FileStatus fileStatus = fileSystem.getFileStatus(path);
System.out.println("*************************************");
System.out.println("檔案根目錄: "+fileStatus.getPath());
System.out.println("這檔案目錄為:");
for(FileStatus fs : fileSystem.listStatus(path)){
System.out.println(fs.getPath());
}
}
//瀏覽檔案
public void look() throws Exception{
Path path = new Path("/user/root/b.txt");
FSDataInputStream fsDataInputStream = fileSystem.open(path);
System.out.println("*************************************");
System.out.println("瀏覽檔案的內容:");
int c;
while((c = fsDataInputStream.read()) != -1){
System.out.print((char)c);
}
fsDataInputStream.close();
}
//下載
public void download() throws Exception{
//下載路徑
String file = "D://hadoop.txt";
InputStream in = fileSystem.open(new Path("/user/root/b.txt"));
OutputStream out = new FileOutputStream(file);
IOUtils.copyBytes(in, out, 4096, true);
System.out.println("*************************************");
System.out.println("檔案下載完成!"+file);
}
//上傳檔案
public void upload() throws Exception{
Path srcPath = new Path("F:/b.txt");
Path dstPath = new Path("/user/root");
fileSystem.copyFromLocalFile(false, srcPath, dstPath);
fileSystem.close();
System.out.println("*************************************");
System.out.println("上傳成功!");
}
//刪除檔案
public void delete() throws Exception{
Path path = new Path("/user/root/a.txt");
fileSystem.delete(path,true);
System.out.println("*************************************");
System.out.println("刪除成功!");
}
3、程式碼的執行
3.1、會出現錯誤 Permission denied: user=Administrator
是因為在windows下執行,預設使用者為 Administrator,該使用者沒有許可權。
3.2、最快速的解決方法
-DHADOOP_USER_NAME=root
新增位置方法如下: