Ganglia环境搭建并监控Hadoop分布式集群(5)

Hadoop环境软件准备

首先需要将JDK以及Hadoop软件解压到需要安装的目录

安装JDK

配置Java到环境变量

验证Java环境是否安装好

安装Hadoop

把Hadoop添加到环境变量

拷贝master01上修改好的/etc/profile文件到其他两台主机

使环境变量立即生效

查看hadoop版本

建立data,name,tmp文件夹：后面配置文件中需要用到

Hadoop配置文件修改

hadoop-env.sh

在文件头添加如下内容

export JAVA_HOME=/usr/local/jdk1.7.0_80
export HADOOP_LOG_DIR=/usr/local/hadoop_repo/logs
export HADOOP_ROOT_LOGGER=DEBUG,DRFA

core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master01:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/usr/local/hadoop_repo/tmp</value>   
</property>
<!--
    <property>
        <name>fs.trash.interval</name>
        <value>1440</value>
    </property>
-->
</configuration>

hdfs-site.xml

<configuration>
    <property> 
        <name>dfs.namenode.name.dir</name> 
        <value>file:///usr/local/hadoop_repo/name</value> 
    </property> 
    <property> 
        <name>dfs.datanode.data.dir</name> 
        <value>file:///usr/local/hadoop_repo/data</value> 
    </property>
    <property>
        <name>dfs.namenode.checkpoint.dir</name>
        <value>file:///usr/local/hadoop_repo/namesecondary</value>
  </property>
  <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>master01:9001</value>
    </property>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property> 
        <name>dfs.webhdfs.enabled</name> 
        <value>true</value> 
    </property><property>
        <name>dfs.permissions</name>
        <value>false</value>
    </property>
</configuration>

mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property> 
        <name>mapreduce.jobhistory.address</name> 
        <value>master01:10020</value> 
    </property> 
    <property> 
        <name>mapreduce.jobhistory.webapp.address</name> 
        <value>master01:19888</value> 
    </property>
    <property>
        <name>yarn.app.mapreduce.am.staging-dir</name>
        <value>/history</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.done-dir</name>
        <value>${yarn.app.mapreduce.am.staging-dir}/history/done</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.intermediate-done-dir</name>
        <value>${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate</value>
    </property>
    <property>
    <name>mapreduce.map.log.level</name>
        <value>DEBUG</value>
    </property>
    <property>
        <name>mapreduce.reduce.log.level</name>
        <value>DEBUG</value>
    </property>
</configuration>

slaves

master01
slave01
slave02

yarn-env.sh

export JAVA_HOME=/usr/local/jdk1.7.0_80
export YARN_LOG_DIR=/usr/local/hadoop_repo/logs
export YARN_ROOT_LOGGER=DEBUG,DRFA

yarn-site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname</name>
    <value>master01</value>
    </property>
    <property> 
        <name>yarn.resourcemanager.address</name> 
        <value>master01:8032</value> 
    </property> 
    <property> 
        <name>yarn.resourcemanager.scheduler.address</name> 
        <value>master01:8030</value> 
    </property> 
    <property> 
        <name>yarn.resourcemanager.resource-tracker.address</name> 
        <value>master01:8031</value> 
    </property> 
    <property> 
        <name>yarn.resourcemanager.admin.address</name> 
        <value>master01:8033</value> 
    </property> 
    <property> 
        <name>yarn.resourcemanager.webapp.address</name> 
        <value>master01:8088</value> 
    </property>
    <property> 
        <name>yarn.log-aggregation-enable</name> 
        <value>true</value> 
    </property>
</configuration>