Commit be83be97db7c3d4a57538b767f148388d1eee594
1 parent
cfb2c9df16
Exists in
master
conf
Showing 1 changed file with 7 additions and 0 deletions Side-by-side Diff
preprocess.sh
View file @
be83be9
| ... | ... | @@ -23,6 +23,13 @@ |
| 23 | 23 | /data/spark/bin/spark-submit \ |
| 24 | 24 | --class com.piki_ds.preprocess.$1 \ |
| 25 | 25 | --master yarn-client \ |
| 26 | +--executor-memory 3000m \ | |
| 27 | +--conf "spark.dynamicAllocation.minExecutors=90" \ | |
| 28 | +--conf "spark.dynamicAllocation.maxExecutors=170" \ | |
| 29 | +--conf "spark.akka.frameSize=2047" \ | |
| 30 | +--conf "spark.network.timeout=120000" \ | |
| 31 | +--conf "spark.shuffle.memoryFraction=0.3" \ | |
| 32 | +--conf "spark.shuffle.manager=hash" \ | |
| 26 | 33 | --conf "spark.default.parallelism=250" \ |
| 27 | 34 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
| 28 | 35 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |