Commit e3f1f7c03fb59092395039144aabba3a8e7a793f

Authored by 박용규 ago
1 parent d89affd590
Exists in master

change deploy-mode to cluster

Showing 6 changed files with 17 additions and 11 deletions Side-by-side Diff

editor_score.sh View file @ e3f1f7c
... ... @@ -15,7 +15,8 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.ver1.EditorScore \
18   ---master yarn-client \
  18 +--master yarn-cluster \
  19 +--conf "spark.yarn.maxAppAttempts=1" \
19 20 --conf "spark.default.parallelism=250" \
20 21 --conf "spark.driver.memory=2g" \
21 22 --conf "spark.dynamicAllocation.minExecutors=90" \
... ... @@ -22,7 +22,8 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class $1 \
25   ---master yarn-client \
  25 +--master yarn-cluster \
  26 +--conf "spark.yarn.maxAppAttempts=1" \
26 27 --executor-memory 3000m \
27 28 --conf "spark.dynamicAllocation.minExecutors=90" \
28 29 --conf "spark.dynamicAllocation.maxExecutors=170" \
makelapse.sh View file @ e3f1f7c
... ... @@ -15,7 +15,8 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.preprocess.MakingLapse \
18   ---master yarn-client \
  18 +--master yarn-cluster \
  19 +--conf "spark.yarn.maxAppAttempts=1" \
19 20 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
20 21 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
21 22 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
mapwithrank.sh View file @ e3f1f7c
... ... @@ -15,7 +15,8 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.preprocess.MapWithRank \
18   ---master yarn-client \
  18 +--master yarn-cluster \
  19 +--conf "spark.yarn.maxAppAttempts=1" \
19 20 --executor-memory 4000m \
20 21 --driver-memory 5000m \
21 22 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
preprocess.sh View file @ e3f1f7c
... ... @@ -22,7 +22,8 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class com.piki_ds.preprocess.$1 \
25   ---master yarn-client \
  25 +--master yarn-cluster \
  26 +--conf "spark.yarn.maxAppAttempts=1" \
26 27 --executor-memory 3000m \
27 28 --conf "spark.driver.memory=2g" \
28 29 --conf "spark.dynamicAllocation.minExecutors=90" \
... ... @@ -22,7 +22,8 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class com.piki_ds.ver1.$1 \
25   ---master yarn-client \
  25 +--master yarn-cluster \
  26 +--conf "spark.yarn.maxAppAttempts=1" \
26 27 --conf "spark.default.parallelism=250" \
27 28 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
28 29 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1