Commit 755e63204a55f41b9d90741fc26b76da872ba04b
1 parent
b356b95668
Exists in
master
preprocess conf change
Showing 1 changed file with 2 additions and 1 deletions Side-by-side Diff
preprocess.sh
View file @
755e632
... | ... | @@ -24,11 +24,12 @@ |
24 | 24 | --class com.piki_ds.preprocess.$1 \ |
25 | 25 | --master yarn-client \ |
26 | 26 | --executor-memory 3000m \ |
27 | +--conf "spark.driver.memory=2g" \ | |
27 | 28 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
28 | 29 | --conf "spark.dynamicAllocation.maxExecutors=170" \ |
29 | 30 | --conf "spark.akka.frameSize=2047" \ |
30 | 31 | --conf "spark.network.timeout=120000" \ |
31 | ---conf "spark.shuffle.memoryFraction=0.3" \ | |
32 | +--conf "spark.shuffle.memoryFraction=0.5" \ | |
32 | 33 | --conf "spark.shuffle.manager=hash" \ |
33 | 34 | --conf "spark.default.parallelism=250" \ |
34 | 35 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |