Hadoop+Hive+HBase+Spark 集群部署(一)

/ 技术分享 / 没有评论 / 2612浏览

节点部署详细信息

parameternodenode1node2node3
typeHadoop : namenode节点; ResourceManager节点;
Spark : master节点;
Hbase : master节点
Hadoop : datanode节点; NodeManager节点
Spark : slave节点;
Hbase : slave节点
Hadoop : datanode节点; NodeManager节点
Spark : slave节点;
Hbase : slave节点
Hadoop : datanode节点; NodeManager节点
Spark : slave节点;
Hbase : slave节点
JAVA_HOME/usr/java/jdk1.8.0_162/usr/java/jdk1.8.0_162/usr/java/jdk1.8.0_162/usr/java/jdk1.8.0_162
HADOOP_HOME/opt/soft/hadoop-2.8.3/opt/soft/hadoop-2.8.3/opt/soft/hadoop-2.8.3/opt/soft/hadoop-2.8.3
HBASE_HOME/opt/soft/hbase-2.0.0/opt/soft/hbase-2.0.0/opt/soft/hbase-2.0.0/opt/soft/hbase-2.0.0
SPARK_HOME/opt/soft/spark-2.3.0-bin-hadoop2.7/opt/soft/spark-2.3.0-bin-hadoop2.7/opt/soft/spark-2.3.0-bin-hadoop2.7/opt/soft/spark-2.3.0-bin-hadoop2.7
SCALA_HOME/opt/soft/scala-2.12.6/opt/soft/scala-2.12.6/opt/soft/scala-2.12.6/opt/soft/scala-2.12.6
ZK_HOME/opt/soft/zookeeper-3.4.12/opt/soft/zookeeper-3.4.12/opt/soft/zookeeper-3.4.12/opt/soft/zookeeper-3.4.12
HIVE_HOME/opt/soft/apache-hive-2.3.3-bin---
MariaDB5.5.56-MariaDB---

环境变量

#Java Config
export JAVA_HOME=/usr/java/jdk1.8.0_162
export JRE_HOME=/usr/java/jdk1.8.0_162/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
# Scala Config
export SCALA_HOME=/opt/soft/scala-2.12.6
# Spark Config
export SPARK_HOME=/opt/soft/spark-2.3.0-bin-hadoop2.7
# Zookeeper Config
export ZK_HOME=/opt/soft/zookeeper-3.4.12
# HBase Config
export HBASE_HOME=/opt/soft/hbase-2.0.0
# Hadoop Config 
export HADOOP_HOME=/opt/soft/hadoop-2.8.3
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
export HADOOP_PREFIX=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_PREFIX
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_PREFIX/lib/native
export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
export HADOOP_HDFS_HOME=$HADOOP_PREFIX
export HADOOP_MAPRED_HOME=$HADOOP_PREFIX
#export HADOOP_ROOT_LOGGER=DEBUG,console
export LD_LIBRARY_PATH=$HADOOP_PREFIX/lib/native
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib:$HADOOP_COMMON_LIB_NATIVE_DIR"
# Hive Config
export HIVE_HOME=/opt/soft/apache-hive-2.3.3-bin
export HIVE_CONF_DIR=${HIVE_HOME}/conf
export PATH=.:${JAVA_HOME}/bin:${SCALA_HOME}/bin:${SPARK_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:${ZK_HOME}/bin:${HBASE_HOME}/bin:${HIVE_HOME}/bin:$PATH

Haoop集群安装