、Hadoop Web專案--HDFS檔案管理
阿新 • • 發佈：2018-12-30
一、專案介紹
推薦系統的web專案已經完成，現在在此基礎上增加HDFS檔案管理功能，便於管理HDFS上的檔案資料，本文基本參考了fansy1990的HDFS檔案管理系統這個專案，改動的地方是將原來分散開的功能集中在了一個頁面和處理了中文顯示亂碼。
二、專案實現
1、開啟選單顯示根目錄檔案及資料夾，點選資料夾進入下一目錄，並可輸入檔名、所有者進行檢索
js程式碼：
java程式碼：$('#dg_hdfsManager_search').datagrid({ border : false, fitColumns : false, singleSelect : true, width : 1050, height : 280, nowrap : false, fit : false, pagination : true,// 分頁控制元件 pageSize : 8, // 每頁記錄數，需要和pageList保持倍數關係 pageList : [ 4, 8, 12 ], rownumbers : true,// 行號 pagePosition : 'top', url : 'hdfs/hdfsManager_searchFolder.action', queryParams: { folder: "/", name:"", nameOp:"no", owner:"", ownerOp:"no" }, onLoadError:function(){ console.info("load error!"); $.messager.alert('警告','讀取錯誤，請聯絡管理員!','warning'); }, onBeforeLoad:function(param){ return checkExistAndAuth(param.folder,'rx'); }, onLoadSuccess:function(data ){ console.info("success,data:"+data); }, idField:'id', columns :[[ { field : 'name', title : '檔名', width : '120', formatter: function(value,row,index){ // 使用轉義即可解決單引號、雙引號不夠用的問題 return "<a href='javascript:void(0);' onclick='refreshDir(\""+value+"\",\""+row.type+"\")"+"'>"+value+"</a>"; }, styler: function(index,row){ var s1=""; if (row.type=='dir'){ return s1 = 'background-color:#FFCCCC;'; } } },{ field : 'type', title : '型別', width : '50', formatter: function(value,row,index){ return "<a href='javascript:void(0);' onclick='refreshDir(\""+row.name+"\",\""+value+"\")"+"'>"+value+"</a>"; } },{ field : 'size', title : '大小', width : '100' },{ field : 'replication', title : '副本數', width : '50', },{ field : 'blockSize', title : '塊大小', width : '100' },{ field : 'modificationTime', title : '修改時間', width : '200' },{ field : 'permission', title : '許可權', width : '150' },{ field : 'owner', title : '所有者', width : '100' },{ field : 'group', title : '組名', width : '100' } ]] });
public void listFolder() throws FileNotFoundException, IllegalArgumentException, IOException { List<HdfsResponseProperties> files = this.hdfsService .listFolder(hdfsFile.getFolder()); Map<String, Object> jsonMap = new HashMap<String, Object>(); jsonMap.put("total", files.size()); jsonMap.put("rows", Utils.getProperFiles(files, page, rows)); Utils.write2PrintWriter(JSON.toJSONString(jsonMap)); return; }
2、新建目錄public List<HdfsResponseProperties> listFolder(String folder) throws FileNotFoundException, IllegalArgumentException, IOException { List<HdfsResponseProperties> files = new ArrayList<>(); FileSystem fs = HadoopUtils.getFs(); FileStatus[] filesStatus = fs.listStatus(new Path(folder)); for (FileStatus file : filesStatus) { files.add(Utils.getDataFromLocatedFileStatus(file)); } return files; }
在目錄輸入框錄入目錄名稱，點選新建目錄按鈕後生成目錄並進入改目錄
主要程式碼
public boolean createFolder(String folder, boolean recursive)
throws IllegalArgumentException, IOException,
AccessControlException {
FileSystem fs = HadoopUtils.getFs();
try {
return fs.mkdirs(new Path(folder));
} catch (AccessControlException e) {
throw e;
}
}
主要程式碼
public boolean upload(String src, String des) throws Exception {
try {
HadoopUtils.getFs().copyFromLocalFile(new Path(src), new Path(des));
} catch (IllegalArgumentException | IOException e) {
log.info("資料上傳異常，src:{},des:{}", new Object[] { src, des });
throw e;
}
return true;
}主要程式碼
public boolean download(String fileName, String localFile) throws Exception {
boolean flag = true;
try {
HadoopUtils.getFs().copyToLocalFile(new Path(fileName),
new Path(localFile));
} catch (Exception e) {
e.printStackTrace();
log.info("資料下載異常，src:{},des:{}",
new Object[] { fileName, localFile });
throw e;
}
return flag;
}js判斷刪除物件是檔案或目錄呼叫不同方法
$('#dg_hdfsManager_folder_delete_btn').bind('click', function(){
var row=$('#dg_hdfsManager_search').datagrid('getSelected');
var value=row.type;
var name=row.name;
var curr =$('#hdfsManager_search_folder').val();
if(curr == '/'){
var folder_=curr+name;
}else{
var folder_=curr+'/'+name;
}
if(value == 'dir'){
var recursive_ ="true";
// ajax 非同步提交任務
var result = callByAJaxHdfs('hdfs/hdfsManager_deleteFolder.action',{folder:folder_,recursive:recursive_});
if("true" == result.flag){
$.messager.alert('資訊','目錄刪除成功!','info');
}else if("false" == result.flag){
$.messager.alert('資訊','目錄刪除失敗，'+result.msg,'info');
}
}else if(value != 'dir'){
var flag = checkExistAndAuth(folder_,'x');
if(!flag) return;
var file_=folder_;
// ajax 非同步提交任務
var result = callByAJaxHdfs('hdfs/hdfsManager_deleteFile.action',{fileName:file_});
if("true" == result.flag){
$.messager.alert('資訊','檔案刪除成功!','info');
}else if("false" == result.flag){
$.messager.alert('資訊','檔案刪除失敗，'+result.msg,'info');
}
}
search_data();
});public boolean deleteFolder(String folder, boolean recursive)
throws IllegalArgumentException, IOException {
FileSystem fs = HadoopUtils.getFs();
try {
return fs.delete(new Path(folder), recursive);
} catch (RemoteException e) {
throw e;
}
}public boolean deleteFile(String fileName) throws Exception {
boolean flag = false;
try {
flag = HadoopUtils.getFs().delete(new Path(fileName), false);
} catch (IllegalArgumentException | IOException e) {
log.info("資料刪除異常，fileName:{}", new Object[] { fileName });
throw e;
}
return flag;
}
主要程式碼
public static String readText(String fileName, int records)
throws IllegalArgumentException, IOException {
FileSystem fs = getFs();
FSDataInputStream inStream = fs.open(new Path(fileName));
BufferedReader br = new BufferedReader(new InputStreamReader(inStream));
StringBuffer buffer = new StringBuffer();
try {
String line;
line = br.readLine();
while (line != null && records-- > 0) {
buffer.append(line).append("<br>");
line = br.readLine();
}
} finally {
br.close();
inStream.close();
}
return buffer.toString();
}三、增加HBASE、HIVE表資料管理
整合HBASE、HIVE表資料管理功能
hive是網上看的，需要先啟動服務 ./bin/hive --service hiveserver，暫時只做了展示表名稱
程式碼：
public void getTables() throws Exception {
Map<String, Object> jsonMap = new HashMap<String, Object>();
int columns = 0;
List<HiveTable> alltables = new ArrayList<>();
Class.forName("org.apache.hadoop.hive.jdbc.HiveDriver");// DriverName 註冊Hive驅動
//建立與Hive資料庫的連線， 預設埠10000，使用資料庫:hive，使用者名稱密碼:hive
// URL,USER,PASSWORD
Connection conn = DriverManager.getConnection("jdbc:hive://192.168.128.129:10000/default", "hh", "");
Statement stmt = conn.createStatement();
String sql="show tables";
ResultSet res = stmt.executeQuery(sql);
while (res.next()) {
System.out.println(res.getString(1));
columns++;
HiveTable hivetable = new HiveTable();
BeanUtils.setProperty(hivetable, "tableName", res.getString(1));
alltables.add(hivetable);
}
conn.close();
conn = null;
jsonMap.put("total", columns);
jsonMap.put("rows", Utils.getProperFiles(alltables, page, rows));
Utils.write2PrintWriter(JSON.toJSONString(jsonMap));
return;
}