Commit 9290e3c54c0a0e9dbac8b68df2cb495030fb88bb
1 parent
b9437aa4d4
Exists in
master
spark-submit conf option tunning
Showing 2 changed files with 8 additions and 4 deletions Side-by-side Diff
editor_score.sh
View file @
9290e3c
... | ... | @@ -17,10 +17,11 @@ |
17 | 17 | --class com.piki_ds.ver1.EditorScore \ |
18 | 18 | --master yarn-cluster \ |
19 | 19 | --conf "spark.yarn.maxAppAttempts=1" \ |
20 | ---conf "spark.default.parallelism=250" \ | |
20 | +--conf "spark.default.parallelism=200" \ | |
21 | 21 | --conf "spark.driver.memory=2g" \ |
22 | ---conf "spark.dynamicAllocation.minExecutors=90" \ | |
23 | ---conf "spark.dynamicAllocation.maxExecutors=170" \ | |
22 | +--conf "spark.executor.memory=4g" \ | |
23 | +--conf "spark.dynamicAllocation.minExecutors=5" \ | |
24 | +--conf "spark.dynamicAllocation.maxExecutors=20" \ | |
24 | 25 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
25 | 26 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
26 | 27 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
score.sh
View file @
9290e3c
... | ... | @@ -24,7 +24,10 @@ |
24 | 24 | --class com.piki_ds.ver1.$1 \ |
25 | 25 | --master yarn-cluster \ |
26 | 26 | --conf "spark.yarn.maxAppAttempts=1" \ |
27 | ---conf "spark.default.parallelism=250" \ | |
27 | +--conf "spark.default.parallelism=120" \ | |
28 | +--conf "spark.executor.memory=3g" \ | |
29 | +--conf "spark.dynamicAllocation.minExecutors=5" \ | |
30 | +--conf "spark.dynamicAllocation.maxExecutors=20" \ | |
28 | 31 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
29 | 32 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
30 | 33 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |