一、準備階段

①在Windows系統上配置Hadoop的環境變數

在path中新增%HADOOP_HOME%\bin
②修改使用者名稱為root，確保對檔案(資料夾)操作時有足夠的許可權

HADOOP_USER_NAME
root

③匯入jar包，並且將core-site.xml和hdfs-site.xml檔案放到src下

二、基於API對HDFS的常用操作

package com.hpe.wangpan.test;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.HdfsBlockLocation;
import org.apache.hadoop.fs.Path;


/**
 * 1、檢視檔案
 * 2、建立新資料夾
 * 3、上傳檔案
 * 4、下載檔案
 * 5、刪除檔案
 * 6、內部移動
 * 7、內部複製
 * 8、重新命名
 * 9、建立新的檔案
 * 10、寫檔案
 * 11、讀檔案內容
 * @author eversec
 *
 */
public class TestHDFS {
	public static void main(String[] args) throws IOException {
		//操作HDFS之前得先建立配置物件
		Configuration conf = new Configuration(true);
		//建立操作HDFS的物件
		FileSystem fs = FileSystem.get(conf);
		
		//檢視檔案系統的內容
		List list = listFileSystem(fs,"/");
		//建立資料夾
//		createDir(fs,"/test/abc");
		
		//上傳檔案
//		uploadFileToHDFS(fs,"d:/wc","/test/abc/");
		
		//下載檔案
//		downLoadFileFromHDFS(fs,"/test/abc/wc","d:/");
		
		//刪除檔案
//		deleteFileFromHDFS(fs,"/test/abc/wc");
		
		//重新命名
//		renameFile(fs,"/test/abc/wc","/test/abc/Angelababy");
		
		//內部移動 內部複製
//		innerCopyAndMoveFile(fs,conf,"/test/abc/Angelababy","/");
		
		//建立一個新檔案
//		createNewFile(fs,"/test/abc/hanhong");
		
		//寫檔案
//		writeToHDFSFile(fs,"/test/abc/hanhong","hello world");
		
		//追加寫
//		appendToHDFSFile(fs,"/test/abc/hanhong","\nhello world");
		
		//讀檔案內容
//		readFromHDFSFile(fs,"/test/abc/hanhong");
		
		//獲取資料的位置
//		getFileLocation(fs,"/install.log");
	}

	//刪除檔案
	private static void deleteFileFromHDFS(FileSystem fs, String string) throws IOException {
		//建立操作HDFS的物件
		Path path =new Path(string);
		fs.deleteOnExit(path);
	}
	
	//獲取資料的位置
	private static void getFileLocation(FileSystem fs, String string) throws IOException {
		FileStatus fileStatus = fs.getFileStatus(new Path(string));
		long len = fileStatus.getLen();
		BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len);
		String[] hosts = fileBlockLocations[0].getHosts();
		for (String string2 : hosts) {
			System.out.println(string2);
		}
		
		HdfsBlockLocation blockLocation = (HdfsBlockLocation)fileBlockLocations[0];
		long blockId = blockLocation.getLocatedBlock().getBlock().getBlockId();
		System.out.println(blockId);
	}
	
	//讀檔案內容
	private static void readFromHDFSFile(FileSystem fs, String string) throws IllegalArgumentException, IOException {
		FSDataInputStream inputStream = fs.open(new Path(string));
		FileStatus fileStatus = fs.getFileStatus(new Path(string));
		long len = fileStatus.getLen();
		byte[] b = new byte[(int)len];
		int read = inputStream.read(b);
		while(read != -1){
			System.out.println(new String(b));
			read = inputStream.read(b);
		}
	}
	//向檔案中追加寫
	private static void appendToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException {
		FSDataOutputStream append = fs.append(new Path(filePath));
		append.write(content.getBytes("UTF-8"));
		append.flush();
		append.close();
	}

	//向檔案中寫內容
	private static void writeToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException {
		 FSDataOutputStream outputStream = fs.create(new Path(filePath));
		 outputStream.write(content.getBytes("UTF-8"));
		 outputStream.flush();
		 outputStream.close();
	}
	
	//建立一個新檔案
	private static void createNewFile(FileSystem fs, String string) throws IllegalArgumentException, IOException {
		fs.createNewFile(new Path(string));
	}

	//內部移動 內部複製
	private static void innerCopyAndMoveFile(FileSystem fs, Configuration conf,String src, String dest) throws IOException {
		Path srcPath = new Path(src);
		Path destPath = new Path(dest);
		
		//內部複製
//		FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,false, conf);
		//內部移動
		FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,true, conf);
	}

	//重新命名
	private static void renameFile(FileSystem fs, String src, String dest) throws IOException {
		Path srcPath = new Path(src);
		Path destPath = new Path(dest);		
		fs.rename(srcPath, destPath);
	}
	
	//下載檔案
	private static void downLoadFileFromHDFS(FileSystem fs, String src, String dest) throws IOException {
		Path srcPath = new Path(src);
		Path destPath = new Path(dest);
		//copyToLocal
//		fs.copyToLocalFile(srcPath, destPath);
		//moveToLocal
		fs.copyToLocalFile(true,srcPath, destPath);
	}

	//上傳檔案
	private static void uploadFileToHDFS(FileSystem fs, String src, String dest) throws IOException {
		Path srcPath = new Path(src);
		Path destPath = new Path(dest);
		//copyFromLocal
//		fs.copyFromLocalFile(srcPath, destPath);
		//moveFromLocal
		fs.copyFromLocalFile(true,srcPath, destPath);
	}

	//建立資料夾
	private static void createDir(FileSystem fs, String string) throws IllegalArgumentException, IOException {
		Path path = new Path(string);
		if(fs.exists(path)){
			fs.delete(path, true);
		}
		fs.mkdirs(path);
	}

	//檢視檔案系統的內容
	private static List listFileSystem(FileSystem fs, String path) throws FileNotFoundException, IOException {
		Path ppath = new Path(path);		
		FileStatus[] listStatus = fs.listStatus(ppath);		
		for (FileStatus fileStatus : listStatus) {
			System.out.println(fileStatus.getPath());
		}		
		return null;
	}
}