Commit 755e63204a55f41b9d90741fc26b76da872ba04b
1 parent
b356b95668
Exists in
master
preprocess conf change
Showing 1 changed file with 2 additions and 1 deletions Side-by-side Diff
preprocess.sh
View file @
755e632
| ... | ... | @@ -24,11 +24,12 @@ |
| 24 | 24 | --class com.piki_ds.preprocess.$1 \ |
| 25 | 25 | --master yarn-client \ |
| 26 | 26 | --executor-memory 3000m \ |
| 27 | +--conf "spark.driver.memory=2g" \ | |
| 27 | 28 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
| 28 | 29 | --conf "spark.dynamicAllocation.maxExecutors=170" \ |
| 29 | 30 | --conf "spark.akka.frameSize=2047" \ |
| 30 | 31 | --conf "spark.network.timeout=120000" \ |
| 31 | ---conf "spark.shuffle.memoryFraction=0.3" \ | |
| 32 | +--conf "spark.shuffle.memoryFraction=0.5" \ | |
| 32 | 33 | --conf "spark.shuffle.manager=hash" \ |
| 33 | 34 | --conf "spark.default.parallelism=250" \ |
| 34 | 35 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |