Commit 755e63204a55f41b9d90741fc26b76da872ba04b

Authored by Joanne ago
1 parent b356b95668
Exists in master

preprocess conf change

Showing 1 changed file with 2 additions and 1 deletions Side-by-side Diff

preprocess.sh View file @ 755e632
... ... @@ -24,11 +24,12 @@
24 24 --class com.piki_ds.preprocess.$1 \
25 25 --master yarn-client \
26 26 --executor-memory 3000m \
  27 +--conf "spark.driver.memory=2g" \
27 28 --conf "spark.dynamicAllocation.minExecutors=90" \
28 29 --conf "spark.dynamicAllocation.maxExecutors=170" \
29 30 --conf "spark.akka.frameSize=2047" \
30 31 --conf "spark.network.timeout=120000" \
31   ---conf "spark.shuffle.memoryFraction=0.3" \
  32 +--conf "spark.shuffle.memoryFraction=0.5" \
32 33 --conf "spark.shuffle.manager=hash" \
33 34 --conf "spark.default.parallelism=250" \
34 35 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1