1. 程式人生 > >Spark叢集搭建+基於zookeeper的高可用HA

Spark叢集搭建+基於zookeeper的高可用HA

export JAVA_HOME=/usr/java/jdk1.8.0_20/

export SCALA_HOME=/home/iespark/hadoop_program_files/scala-2.10.6/

export HADOOP_HOME=/home/iespark/hadoop_program_files/hadoop-2.6.0

export HADOOP_CONF_DIR=/home/iespark/hadoop_program_files/hadoop-2.6.0/etc/hadoop

#export SPARK_MASTER_IP=hadoop5(設定了zookeeper這行不用了,註釋掉)

export SPARK_DAEMON_JAVA_OPTS=”-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=master:2181,slave1:2181,slave2:2181 –Dspark.deploy.zookeeper.dir=/spark”

export SPARK_WORKER_MEMORY=2g

export SPARK_EXECUTOR_MEMORY=2g

export SPARK_DRIVER_MEMORY=2g

export SPARK_WORKER_CORES=1

#export SPARK_PID_DIR=/home/iespark/hadoop_program_files/sparkdata