Spark叢集搭建+基於zookeeper的高可用HA
阿新 • • 發佈:2018-12-25
export JAVA_HOME=/usr/java/jdk1.8.0_20/
export SCALA_HOME=/home/iespark/hadoop_program_files/scala-2.10.6/
export HADOOP_HOME=/home/iespark/hadoop_program_files/hadoop-2.6.0
export HADOOP_CONF_DIR=/home/iespark/hadoop_program_files/hadoop-2.6.0/etc/hadoop
#export SPARK_MASTER_IP=hadoop5(設定了zookeeper這行不用了,註釋掉)
export SPARK_DAEMON_JAVA_OPTS=”-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=master:2181,slave1:2181,slave2:2181 –Dspark.deploy.zookeeper.dir=/spark”
export SPARK_WORKER_MEMORY=2g
export SPARK_EXECUTOR_MEMORY=2g
export SPARK_DRIVER_MEMORY=2g
export SPARK_WORKER_CORES=1
#export SPARK_PID_DIR=/home/iespark/hadoop_program_files/sparkdata