上傳並解壓壓縮包

Hadoop下載

Hadoop配置

配置的檔案目錄是hadoop安裝目錄下的etc/bin/hadoop

core-site.xml

<configuration>


<!-- 臨時檔案儲存目錄   -->
   <property>

        <name>hadoop.tmp.dir</name>

        <value>自己指定</value>

    </property>

<!-- namenode指定，指定檔案系統型別：分散式檔案系統 -->

    <property>

        <name>fs.default.name</name>

        <value>hdfs://192.168.2.135:9000</value>

    </property>
<!-- 緩衝區大小  -->
    <property>
        <name>io.file.buffer.size</name>
        <value>4096</value>
    </property>

<!-- 開啟hdfs垃圾桶機制，單位分鐘  -->
    <property>
        <name>fs.trash.interval</name>
        <value>10080</value>
    </property>

</configuration>

 

hdfs-site.xml

<configuration>
    <!-- 指定 Hadoop 輔助名稱節點主機配置-->
        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>node01:50090</value>
        </property>
        <!-- 指定namenode訪問位置-->
        <property>
                <name>dfs.namenode.http-address</name>
                <value>node01:50070</value>
        </property>

  <!-- 配置 HDFS 的備份檔案數量, 預設數量是3 -->
        <property>
                <name>dfs.replication</name>
                <value>3</value>
        </property>
       

<!-- 指定datanode資料的存放位置-->
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>file:///export/servers/hadoop-2.7.5/hadoopDatas/datanodeDatas,
                file:///export/servers/hadoop-2.7.5/hadoopDatas/datanodeDatas2</value>
        </property>

<!-- 指定namenode元資料存放位置,元資料是資料的目錄-->
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>file:///export/servers/hadoop-2.7.5/hadoopDatas/namenodeDatas,
                file:///export/servers/hadoop-2.7.5/hadoopDatas/namenodeDatas2</value>
        </property>
        
        
         <property>
                <name>dfs.namenode.edits.dir</name>
                <value>file:///export/servers/hadoop-2.7.5/hadoopDatas/nn/edits</value>
        </property>
        
        
        <property>
                <name>dfs.namenode.checkpoint.dir</name>
                <value>file:///export/servers/hadoop-2.7.5/hadoopDatas/snn/name</value>
        </property>
        
        <property>
                <name>dfs.namenode.checkpoint.edits.dir</name>
                <value>file:///export/servers/hadoop-2.7.5/hadoopDatas/dfs/snn/edits</value>
        </property>
        
        
        <property>
                <name>dfs.permissions</name>
                <value>false</value>
        </property>
        
        <property>
                <name>dfs.blocksize</name>
                <value>34217728</value>
        </property>

        
</configuration>

 

hadoop-env.sh(jdk配置)

export JAVA_HOME=/export/servers/jdk1.8.0_271/

mapred-site.xml

<configuration>


        <!-- 開啟MapReduce小任務模式  -->
        <property>
                <name>mapreduce.job.ubertask.enable</name>
                <value>true</value>
        </property>



        <!-- 設定歷史任務主機和埠  -->
        <property>
                <name>mapreduce.jobhistory.address</name>
                <value>node01:10020</value>
        </property>

        <!-- 設定網頁訪問歷史任務的主機和埠  -->
        <property>
                <name>mapreduce.jobhistory.webapp.address</name>
                <value>node01:19888</value>
        </property>

</configuration>

 

yarn-site.xml

<configuration>
<!-- Site specific YARN configuration properties -->
<!--  指定yarn執行的主節點  -->
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>node01</value>
		</property>
   
<!--  指定yarn執行的主機  -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
		</property>
   
   <!--  開啟日誌聚合功能  -->
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
		</property>

<!--  聚合日誌儲存時間  -->
    <property>
        <name>yarn.resourcemanager.retain-seconds</name>
        <value>604800</value>
		</property>

<!--  設定yarn叢集的記憶體分配方案  -->
    <property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>20480</value>
		</property>


    <property>
        <name>yarn.scheduler.minimunm-allocation-mb</name>
        <value>2048</value>
		</property>
   
    <property>
        <name>yarn.nodemanager.vmem-pmem-ratio</name>
        <value>2.1</value>
		</property>

</configuration>

mapred-env.sh

export JAVA_HOME=/export/servers/jdk1.8.0_271/

slaves

#從節點配置
node01
node02
node03

配置hadoop環境變數

#hadoop環境變數
export HADOOP_HOME=/export/servers/hadoop-2.7.5
export PATH=:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH