Commit be83be97db7c3d4a57538b767f148388d1eee594

Authored by Joanne ago
1 parent cfb2c9df16
Exists in master

conf

Showing 1 changed file with 7 additions and 0 deletions Side-by-side Diff

preprocess.sh View file @ be83be9
... ... @@ -23,6 +23,13 @@
23 23 /data/spark/bin/spark-submit \
24 24 --class com.piki_ds.preprocess.$1 \
25 25 --master yarn-client \
  26 +--executor-memory 3000m \
  27 +--conf "spark.dynamicAllocation.minExecutors=90" \
  28 +--conf "spark.dynamicAllocation.maxExecutors=170" \
  29 +--conf "spark.akka.frameSize=2047" \
  30 +--conf "spark.network.timeout=120000" \
  31 +--conf "spark.shuffle.memoryFraction=0.3" \
  32 +--conf "spark.shuffle.manager=hash" \
26 33 --conf "spark.default.parallelism=250" \
27 34 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
28 35 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1