Eclipse+hadoop偽態式分佈+API
阿新 • • 發佈：2021-07-13
Eclipse+hadoop偽態式分佈+API
1、在eclipse中新建一個Maven檔案，下載對應jar包
<dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>2.9.2</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.9.2</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.9.2</version> </dependency>
2、編寫指令碼程式碼（瀏覽hdfs目錄）
public class Test { public static void main(String[] args) throws IOException { // TODO Auto-generated method stub String dir = "hdfs://192.168.80.128:9000/"; try { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(dir),conf); FileStatus[] status = fs.listStatus(new Path(dir)); List<String> names = new ArrayList<String>(); for (int i = 0; i < status.length; ++i) { if (status[i].isFile()) { names.add(status[i].getPath().toString()); System.out.println(status[i].getPath().toString()); }else if (status[i].isDirectory()) { names.add(status[i].getPath().toString()); System.out.println(status[i].getPath().toString()); } } fs.close(); } catch (Exception e) { // TODO: handle exception } } }
3、如果出現一下報錯
在src下建立檔案log4j.properties
log4j.rootLogger=WARN, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
4、即可使用java程式設計來對hadoop進行操作
API操作
1、瀏覽hdfs目錄檔案（資料夾和檔案）
public class ListFileAndFolder {
public static void main(String[] args) throws IOException {
// TODO Auto-generated method stub
String dir = "hdfs://192.168.80.128:9000/";//地址
try {
Configuration conf = new Configuration();//載入配置檔案
FileSystem fs = FileSystem.get(URI.create(dir),conf);//獲取檔案系統例項
FileStatus[] status = fs.listStatus(new Path("/test"));//待獲取目錄
List<String> names = new ArrayList<String>();//存放獲取的目錄連結
for (int i = 0; i < status.length; ++i) {
if (status[i].isFile()) {//檔案
names.add(status[i].getPath().toString());
System.out.println(status[i].getPath().toString());
}else if (status[i].isDirectory()) {//資料夾
names.add(status[i].getPath().toString());
System.out.println(status[i].getPath().toString());
}
}
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失敗");
}
}
}
2、建立資料夾
public class CreateFolder {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";//埠9000
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//使用root使用者才有許可權
String pathString = "/wujf";//資料夾名
boolean exists = fs.exists(new Path(pathString));
if(!exists){//資料夾不存在則建立
boolean result = fs.mkdirs(new Path(pathString));
System.out.println(result);
}
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失敗");
}
}
}
3、建立檔案並且寫入內容
public class WriteFile {
public static void main(String[] args) {
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root使用者
String pathString = "/wujf/wujf"+".txt";//路徑以及檔名
FSDataOutputStream fsDataOutputStream = fs.create(new Path(pathString));
String inpuString = "I am wujf";//寫入的內容
IOUtils.copyBytes(new ByteArrayInputStream(inpuString.getBytes()),fsDataOutputStream,conf,true);
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失敗");
}
}
}
4、複製檔案
public class CopyFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
//root使用者才有許可權建立檔案
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
String pathString = "/output/write1"+".txt";//複製後的檔案路徑以及檔名
FSDataOutputStream fsDataOutputStream = fs.create(new Path(pathString));
String inpuString = "/input/test1.txt";//待複製的檔案
IOUtils.copyBytes(new ByteArrayInputStream(inpuString.getBytes()),fsDataOutputStream,conf,true);
//引數1：輸入流，引數2：輸出流；引數3：配置物件，引數4：是否關閉流
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.println("失敗");
}
}
}
5、讀取檔案內容
public class ReadFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
String pathString = "/output/write2"+".txt";//待讀取的檔案
FSDataInputStream fsDataInputStream = fs.open(new Path(pathString));
IOUtils.copyBytes(fsDataInputStream, System.out, conf ,true);
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.print("失敗");
}
}
}
6、刪除檔案
public class DeleteFolder {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");
String pathString = "/output/write"+".txt";
System.out.print(fs.deleteOnExit(new Path(pathString)));
fs.close();
} catch (Exception e) {
// TODO: handle exception
System.out.print("失敗");
}
}
}
7、上傳本地檔案
public class UploadFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root使用者許可權高，不容易出現其他問題
Path srcPath = new Path("D:\\temp\\abc.txt");//本地檔案路徑
Path upPath=new Path("/");//上傳檔案的儲存位置,存放於根目錄
fs.copyFromLocalFile(srcPath, upPath);
} catch (Exception e) {
// TODO: handle exception
System.out.println("失敗");
}
}
}
8、下載檔案到本地
public class DownloadFile {
public static void main(String[] args) {
// TODO Auto-generated method stub
String uri = "hdfs://192.168.80.128:9000/";
try {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(uri),conf,"root");//root使用者許可權高，不容易出現其他問題
Path inputStream = new Path("/write.txt");//待下載檔案
Path outputStream = new Path("D:\\temp");//存放路徑
//引數1：是否刪除原始檔，引數2：待下載檔案路徑，引數3：存放路徑，引數4:是否開啟檔案校驗
fs.copyToLocalFile(false, inputStream, outputStream, true);
} catch (Exception e) {
// TODO: handle exception
System.out.println("失敗");
}
}
}