spark standalone
spark-1.5.0-bin-hadoop2.6
spark-env.sh配置
export SPARK_MASTER_IP=172.16.31.11
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_CORES=1
export SPARK_WORDER_INSTANCES=1
export SPARK_WORKER_MEMORY=1g
export MASTER=spark://172.16.31.11:7077
slaves
MRguan
啟動
./sbin/start-all.sh
./bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode client lib/spark-examples-1.5.0-hadoop2.6.0.jar
spark sql with hive
spark-env.sh添加
export HADOOP_CONF_DIR=/opt/beh/core/hadoop/etc/hadoop
export YARN_CONF_DIR=/opt/beh/core/hadoop/etc/hadoop
export HADOOP_HOME=/opt/beh/core/hadoop
export SPARK_JAR=/opt/beh/core/spark/lib/spark-assembly-1.5.0-hadoop2.6.0.jar
export SPARK_CLASSPATH=/opt/beh/core/spark/lib/mysql-connector-java-5.1.30-bin.jar
?將hive-site.xml和hive-log4j.properties復制到$SPARK_HOME/conf下
?將 $HIVE_HOME/lib下 mysql-connector-java-5.1.30-bin.jar復制到$SPARK_HOME/lib下
注意版本 不然報錯
17/02/28 09:12:48 WARN HiveMetaStore: Retrying creating default database after error: Error creating transactional connection factory
javax.jdo.JDOFatalInternalException: Error creating transactional connection factory
