軟體版本：hadoop2.2，JDK1.7，Intellij idea14

0. 配置Configuration

如果需要使用java程式來獲得某個叢集的NameNode或DataNode的相關資訊，那麼Configuration需要如下配置：

/**
	 * 獲得配置的Configuration
	 * 
	 * @return
	 */
	public static Configuration getConf() {
		if (conf == null) {
			conf = new Configuration();
			conf.setBoolean("mapreduce.app-submission.cross-platform", true);// 配置使用跨平臺提交任務
			conf.set("fs.defaultFS", "hdfs://" + namenode + ":8020");// 指定namenode
			conf.set("mapreduce.framework.name", "yarn"); // 指定使用yarn框架
			conf.set("yarn.resourcemanager.address", resourcenode + ":8032"); // 指定resourcemanager
			conf.set("yarn.resourcemanager.scheduler.address", schedulernode + ":8030");// 指定資源分配器
			conf.set("mapreduce.jobhistory.address", jobhistorynode + ":10020");// 指定historyserver
		}
		return conf;
	}
 

1. 針對Hadoop叢集（不管有沒有配置HA），獲得DataNode相關資訊；

/**
     * 獲取子節點
     * @return
     */
    public static DatanodeInfo getDATANODE() {
        if(DATANODE==null) {
            try {
                DatanodeInfo[] live =  ((DistributedFileSystem) getFS()).getDataNodeStats(HdfsConstants.DatanodeReportType.LIVE);
                if(live.length<1){
                    log.error("獲取live子節點失敗!");
                }
                DATANODE=live[0];
            }catch (IOException e){
                log.warn("獲取live子節點異常！\n"+e.getMessage());
            }
        }
        return DATANODE;
    }
 

package org.apache.hadoop.hdfs.tools;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HAServiceTarget;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

/**
 * 動態獲得Active NameNode
 * 針對HA叢集
 * Created by fansy on 2016/3/29.
 */
public class DFSHAAdminUtils extends DFSHAAdmin {
    private Logger log = LoggerFactory.getLogger(DFSHAAdminUtils.class);
    private int rpcTimeoutForChecks = -1;
    private  String nnidStr;
    private static String activeNameNodeHostAndPort=null;
    public DFSHAAdminUtils(){}

    public DFSHAAdminUtils(Configuration conf,String nnidStr){
        super.setConf(conf);
        if (conf != null) {
            rpcTimeoutForChecks = conf.getInt(
                    CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY,
                    CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT);
        }
        this.nnidStr= nnidStr;
    }

    public String getActiveNameNode(){
        String[] nnids = nnidStr.split(",",-1);
        HAServiceProtocol proto = null;
        HAServiceTarget haServiceTarget=null;
        HAServiceStatus state= null;

        try {
            for (String nnid : nnids){
                haServiceTarget = resolveTarget(nnid);
                proto = haServiceTarget.getProxy(
                        getConf(), rpcTimeoutForChecks);
                state = proto.getServiceStatus();
                if (state.getState().ordinal()==HAServiceProtocol.HAServiceState.ACTIVE.ordinal()) {
                    log.info("Active NameNode:{}",
                            haServiceTarget.getAddress().getHostName() + ":" + haServiceTarget.getAddress().getPort());
                    return haServiceTarget.getAddress().getHostName()+":"+haServiceTarget.getAddress().getPort();
                }
            }
        }catch(IOException e){
            log.error("獲取Active NameNode異常!");
        }
        return null;
    }
}
 

其中，DFSHAAdmin是Hadoop自帶的，同時，這裡的實現也是參考了其內部的實現；

分享，成長，快樂

腳踏實地，專注