Commit 9290e3c54c0a0e9dbac8b68df2cb495030fb88bb

Authored by 박용규 ago
1 parent b9437aa4d4
Exists in master

spark-submit conf option tunning

Showing 2 changed files with 8 additions and 4 deletions Side-by-side Diff

editor_score.sh View file @ 9290e3c
... ... @@ -17,10 +17,11 @@
17 17 --class com.piki_ds.ver1.EditorScore \
18 18 --master yarn-cluster \
19 19 --conf "spark.yarn.maxAppAttempts=1" \
20   ---conf "spark.default.parallelism=250" \
  20 +--conf "spark.default.parallelism=200" \
21 21 --conf "spark.driver.memory=2g" \
22   ---conf "spark.dynamicAllocation.minExecutors=90" \
23   ---conf "spark.dynamicAllocation.maxExecutors=170" \
  22 +--conf "spark.executor.memory=4g" \
  23 +--conf "spark.dynamicAllocation.minExecutors=5" \
  24 +--conf "spark.dynamicAllocation.maxExecutors=20" \
24 25 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
25 26 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
26 27 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
... ... @@ -24,7 +24,10 @@
24 24 --class com.piki_ds.ver1.$1 \
25 25 --master yarn-cluster \
26 26 --conf "spark.yarn.maxAppAttempts=1" \
27   ---conf "spark.default.parallelism=250" \
  27 +--conf "spark.default.parallelism=120" \
  28 +--conf "spark.executor.memory=3g" \
  29 +--conf "spark.dynamicAllocation.minExecutors=5" \
  30 +--conf "spark.dynamicAllocation.maxExecutors=20" \
28 31 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
29 32 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
30 33 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1