hadoop@hadoop-pc / $ java -version java version "1.8.0_92" Java(TM) SE Runtime Environment (build 1.8.0_92-b14) Java HotSpot(TM) 64-Bit Server VM (build 25.92-b14, mixed mode) hadoop@hadoop-pc / $ echo $JAVA_HOME /opt/lib/jdk8 hadoop@hadoop-pc / $ echo $CLASSPATH .:/opt/lib/jdk8/lib/dt.jar:/opt/lib/jdk8/lib/tools.jar hadoop@hadoop-pc / $ echo $PATH /opt/lib/jdk8/bin:/opt/lib/jdk8/bin:/opt/lib/jdk8/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games hadoop@hadoop-pc / $
建立几个hadoop需要用的文件夹
tmp目录
mkdir /opt/hadoop-tmp
hdfs目录
mkdir /opt/hadoop-dfs
name 目录
mkdir /opt/hadoop-dfs/name
data目录
mkdir /opt/hadoop-dfs/data
上传hadoop
用MX把hadoop的压缩包上传到/opt,或者在/opt下 wget
tar -zxvf hadoop-2.7.1.tar.gz
mv hadoop-2.7.1.tar.gz lib/package/ 把压缩包备份到package
mv hadoop-2.7.1 hadoop 重命名一下文件夹
修改一下hadoop的配置文件
hadoop 的配置文件在/opt/hadoop/etc/hadoop下面
core-site.xml
<configuration> <property> <name>fs.defaultFS</name> <value>hdfs://master:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>file:/opt/hadoop-tmp</value> <description>Abasefor other temporary directories.</description> </property> </configuration>
hdfs-site.xml
<configuration> <property> <name>dfs.namenode.secondary.http-address</name> <value>master:9001</value> </property> <property> <name>dfs.namenode.name.dir</name> <value>file:/opt/hadoop-dfs/name</value> </property> <property> <name>dfs.datanode.data.dir</name> <value>file:/opt/hadoop-dfs/data</value> </property> <property> <name>dfs.replication</name> <value>3</value> </property> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> </property> </configuration>
mapred-site.xml
cp mapred-site.xml.template mapred-site.xml
<configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> <property> <name>mapreduce.jobhistory.address</name> <value>master:10020</value> </property> <property> <name>mapreduce.jobhistory.webapp.address</name> <value>master:19888</value> </property> </configuration>
yarn-site.xml
<configuration> <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> <property> <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> <value>org.apache.hadoop.mapred.ShuffleHandler</value> </property> <property> <name>yarn.resourcemanager.address</name> <value>master:8032</value> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> <value>master:8030</value> </property> <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>master:8035</value> </property> <property> <name>yarn.resourcemanager.admin.address</name> <value>master:8033</value> </property> <property> <name>yarn.resourcemanager.webapp.address</name> <value>master:8088</value> </property> </configuration>
slaves
1
2
slave1
slave2
hadoop-env.sh
修改JAVA_HOME
export JAVA_HOME=/opt/lib/jdk8
yarn-env.sh
添加JAVA_HOME环境变量
export JAVA_HOME=/opt/lib/jdk