Commit 9290e3c54c0a0e9dbac8b68df2cb495030fb88bb
1 parent
b9437aa4d4
Exists in
master
spark-submit conf option tunning
Showing 2 changed files with 8 additions and 4 deletions Side-by-side Diff
editor_score.sh
View file @
9290e3c
| ... | ... | @@ -17,10 +17,11 @@ |
| 17 | 17 | --class com.piki_ds.ver1.EditorScore \ |
| 18 | 18 | --master yarn-cluster \ |
| 19 | 19 | --conf "spark.yarn.maxAppAttempts=1" \ |
| 20 | ---conf "spark.default.parallelism=250" \ | |
| 20 | +--conf "spark.default.parallelism=200" \ | |
| 21 | 21 | --conf "spark.driver.memory=2g" \ |
| 22 | ---conf "spark.dynamicAllocation.minExecutors=90" \ | |
| 23 | ---conf "spark.dynamicAllocation.maxExecutors=170" \ | |
| 22 | +--conf "spark.executor.memory=4g" \ | |
| 23 | +--conf "spark.dynamicAllocation.minExecutors=5" \ | |
| 24 | +--conf "spark.dynamicAllocation.maxExecutors=20" \ | |
| 24 | 25 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
| 25 | 26 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
| 26 | 27 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
score.sh
View file @
9290e3c
| ... | ... | @@ -24,7 +24,10 @@ |
| 24 | 24 | --class com.piki_ds.ver1.$1 \ |
| 25 | 25 | --master yarn-cluster \ |
| 26 | 26 | --conf "spark.yarn.maxAppAttempts=1" \ |
| 27 | ---conf "spark.default.parallelism=250" \ | |
| 27 | +--conf "spark.default.parallelism=120" \ | |
| 28 | +--conf "spark.executor.memory=3g" \ | |
| 29 | +--conf "spark.dynamicAllocation.minExecutors=5" \ | |
| 30 | +--conf "spark.dynamicAllocation.maxExecutors=20" \ | |
| 28 | 31 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
| 29 | 32 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
| 30 | 33 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |