操作HDFS叢集的JavaAPI
阿新 • • 發佈:2018-12-15
一、準備階段
首先配置本機的環境變數,配置好HADOOP_HOME;在eclipse中安裝好外掛,以便在eclipse中操作HDFS叢集,將hadoop-eclipse-plugin-2.6.0.jar包放入到eclipse安裝資料夾下的dropins資料夾中的plugins下,此後重啟eclipse即可在eclipse中操作hdfs叢集。 注意: 編寫的程式碼若有操作檔案(例如修改、刪除等功能)時,需要在本機配置許可權的環境變數:HADOOP_USER_NAME=root,這樣才會避免在eclipse中出現許可權問題。
二、eclipse中的配置
環境變數配置好後,在eclipse中完成配置即可操作hdfs叢集: 1)eclipse中的Hadoop控制平臺
2)配置好所要連線的namenodeIP地址 此前要修改本機的hosts檔案來確定IP地址對應的主機名 3)操作 4)替換工程對應的Hadoop包下的bin資料夾,使其方便操作 5)匯入工程必需的jar包
三、JavaAPI程式碼
/** * 1、檢視檔案 * 2、建立新資料夾 * 3、上傳檔案 * 4、下載檔案 * 5、刪除檔案 * 6、內部移動 * 7、內部複製 * 8、重新命名 * 9、建立新的檔案 * 10、寫檔案 * 11、讀檔案內容 * */ //操作HDFS之前得先建立配置物件 Configuration conf = new Configuration(true); //建立操作HDFS的物件 FileSystem fs = FileSystem.get(conf);
功能API程式碼:
//獲取資料的位置 private static void getFileLocation(FileSystem fs, String string) throws IOException { FileStatus fileStatus = fs.getFileStatus(new Path(string)); long len = fileStatus.getLen(); BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, len); String[] hosts = fileBlockLocations[0].getHosts(); for (String string2 : hosts) { System.out.println(string2); } HdfsBlockLocation blockLocation = (HdfsBlockLocation)fileBlockLocations[0]; long blockId = blockLocation.getLocatedBlock().getBlock().getBlockId(); System.out.println(blockId); } //讀檔案內容 private static void readFromHDFSFile(FileSystem fs, String string) throws IllegalArgumentException, IOException { FSDataInputStream inputStream = fs.open(new Path(string)); FileStatus fileStatus = fs.getFileStatus(new Path(string)); long len = fileStatus.getLen(); byte[] b = new byte[(int)len]; int read = inputStream.read(b); while(read != -1){ System.out.println(new String(b)); read = inputStream.read(b); } } //寫檔案 private static void writeToHDFSFile(FileSystem fs, String filePath, String content) throws IllegalArgumentException, IOException { FSDataOutputStream outputStream = fs.create(new Path(filePath)); outputStream.write(content.getBytes("UTF-8")); outputStream.flush(); outputStream.close(); } //建立一個新檔案 private static void createNewFile(FileSystem fs, String string) throws IllegalArgumentException, IOException { fs.createNewFile(new Path(string)); } //內部移動 內部複製 private static void innerCopyAndMoveFile(FileSystem fs, Configuration conf,String src, String dest) throws IOException { Path srcPath = new Path(src); Path destPath = new Path(dest); //內部拷貝 FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,false, conf); //內部移動 FileUtil.copy(srcPath.getFileSystem(conf), srcPath, destPath.getFileSystem(conf), destPath,true, conf); } //重新命名 private static void renameFile(FileSystem fs, String src, String dest) throws IOException { Path srcPath = new Path(src); Path destPath = new Path(dest); fs.rename(srcPath, destPath); } //下載檔案 private static void downLoadFileFromHDFS(FileSystem fs, String src, String dest) throws IOException { Path srcPath = new Path(src); Path destPath = new Path(dest); //copyToLocal fs.copyToLocalFile(srcPath, destPath); //moveToLocal fs.copyToLocalFile(true,srcPath, destPath); } //上傳檔案 private static void uploadFileToHDFS(FileSystem fs, String src, String dest) throws IOException { Path srcPath = new Path(src); Path destPath = new Path(dest); //copyFromLocal fs.copyFromLocalFile(srcPath, destPath); //moveFromLocal fs.copyFromLocalFile(true,srcPath, destPath); } //建立資料夾 private static void createDir(FileSystem fs, String string) throws IllegalArgumentException, IOException { Path path = new Path(string); if(fs.exists(path)){ fs.delete(path, true); } fs.mkdirs(path); } //檢視檔案系統的內容 private static List listFileSystem(FileSystem fs, String path) throws FileNotFoundException, IOException { Path ppath = new Path(path); FileStatus[] listStatus = fs.listStatus(ppath); for (FileStatus fileStatus : listStatus) { System.out.println(fileStatus.getPath()); } return null; } //刪除檔案 private static String del(String path) throws IOException{ Configuration conf=new Configuration(true); FileSystem fs = FileSystem.get(conf); Path ppath=new Path(path); try { fs.delete(ppath); fs.close(); } catch (IOException e) { e.printStackTrace(); } System.out.println("刪除檔案:"+ppath); }