(6)Java 讀寫 hdfs檔案或者目錄
阿新 • • 發佈:2019-01-31
1.讀取單個檔案
Date date = DateUtil.getSpecifiedDayBefore(); String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd"); String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(path), conf); FSDataInputStream hdfsInStream = fs.open(new Path(path)); InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8"); BufferedReader br = new BufferedReader(isr); String line; // int k = 0; while ((line = br.readLine()) != null) { System.out.println(line); }
2.讀取資料夾
Date date = DateUtil.getSpecifiedDayBefore(); String yesterday = DateUtil.dateToStr(date, "yyyy-MM-dd"); String path = "hdfs://ip:9000/output_log/output_log_click" + yesterday; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(path), conf); FileStatus[] status = fs.listStatus(new Path(path)); for (FileStatus file : status) { if (!file.getPath().getName().startsWith("newsMap")) { continue; } FSDataInputStream hdfsInStream = fs.open(file.getPath()); InputStreamReader isr = new InputStreamReader(hdfsInStream, "utf-8"); BufferedReader br = new BufferedReader(isr); String line; // int k = 0; while ((line = br.readLine()) != null) { System.out.println(line); } }