linux上學習pyspark執行程式的命令
阿新 • • 發佈:2018-12-14
2002 spark-submit --master spark://192.168.0.80:7077 /home/tsl/chenchangsha/test_spark_run_model/temp_1.py 2003 : java.lang.NullPointerException 2004 echo $JAVA_HOME 2005 /usr/lib/jdk1.8.0_181 2006 echo $SCALA_HOME 2007 echo $HADOOP_HOME 2008 /home/tsl/changsha/hadoop-2.6.5 /home/tsl/changsha/hadoop-2.6. 2009 vim ~/conf/spark_env.sh 2010 sudo vim ~/conf/spark_env.sh 2011 sudo chown tsl:~/conf/spark_env.sh 2012 sudo chown tsl ~/conf/spark_env.sh 2013 ll 2014 cd changsha 2015 ll 2016 cd spark-2.3.2-bin-hadoop2.6/ 2017 ll 2018 cd bin 2019 sudo chown tsl ./conf/spark_env.sh 2020 ll 2021 cd ./ 2022 cd .. 2023 sudo chown tsl ./conf/spark_env.sh 2024 cd conf/ 2025 ll 2026 sudo chown tsl ./spark-env.sh 2027 sudo vim ./spark_env.sh 2028 spark-submit --master spark://192.168.0.80:7077 /home/tsl/chenchangsha/test_spark_run_model/temp_1.py 2029 history 1948 ll 1949 pwd 1950 cd home 1951 cd home/ 1952 cd changsha/ 1953 ll 1954 tar -zxvf jdk-8u181-linux-x64.tar.gz 1955 sudo mv jdk-8u181 /usr/lib 1956 sudo mv jdk1.8.0_181 /usr/lib 1957 tar -zxvf scala-2.12.7.tgz 1958 sudo mv scala-2.12.7 /usr/lib 1959 tar -zxvf hadoop-2.6.5.tar.gz 1960 vim ~/.profile 1961 source ~/.profile 1962 vim hadoop-2.6.5/etc/hadoop/core-site.xml 1963 vim hadoop-2.6.5/etc/hadoop/mapred-site.xml 1964 vim hadoop-2.6.5/etc/hadoop/hdfs-site.xml 1965 vim hadoop-2.6.5/etc/hadoop/hadoop-env.sh 1966 vim ./spark-2.3.2-bin-hadoop2.6/conf/spark-env.sh 1967 sudo vim ./spark-2.3.2-bin-hadoop2.6/conf/spark-env.sh 1968 tar -zxvf ./spark-2.3.2-bin-hadoop2.6.tgz 1969 sudo vim ./spark-2.3.2-bin-hadoop2.6/conf/spark-env.sh 1970 java 1971 HADOOP_HOME/bin/hdfs namenode -format 1972 HADOOP_HOME/sbin/start-all.sh 1973 pyspark 1974 sudo apt-get install ssh openssh-server 1975 ssh-keygen -t rsa -P "" 1976 cat ~/.ssh/id_rsa.pub 1977 ~/.ssh/authorized_keys 1978 service ssh start 1979 HADOOP_HOME/bin/hdfs namenode -format 1980 HADOOP_HOME/sbin/start-all.sh 1981 ./start-all.sh 1982 echo $HADOOP_HOME 1983 vim ~/.profile 1984 vim ~/.bashrc 1985 source ~/.bashrc 1986 sudo vim ~/.bashrc