Commit 1e44a601b71039d341ad08312378001cb7b21504
1 parent
e3f1f7c03f
Exists in
master
deploy-mode revert
Showing 6 changed files with 6 additions and 12 deletions Side-by-side Diff
editor_score.sh
View file @
1e44a60
... | ... | @@ -15,8 +15,7 @@ |
15 | 15 | |
16 | 16 | /data/spark/bin/spark-submit \ |
17 | 17 | --class com.piki_ds.ver1.EditorScore \ |
18 | ---master yarn-cluster \ | |
19 | ---conf "spark.yarn.maxAppAttempts=1" \ | |
18 | +--master yarn-client \ | |
20 | 19 | --conf "spark.default.parallelism=250" \ |
21 | 20 | --conf "spark.driver.memory=2g" \ |
22 | 21 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
gghVer2.sh
View file @
1e44a60
... | ... | @@ -22,8 +22,7 @@ |
22 | 22 | |
23 | 23 | /data/spark/bin/spark-submit \ |
24 | 24 | --class $1 \ |
25 | ---master yarn-cluster \ | |
26 | ---conf "spark.yarn.maxAppAttempts=1" \ | |
25 | +--master yarn-client \ | |
27 | 26 | --executor-memory 3000m \ |
28 | 27 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
29 | 28 | --conf "spark.dynamicAllocation.maxExecutors=170" \ |
makelapse.sh
View file @
1e44a60
... | ... | @@ -15,8 +15,7 @@ |
15 | 15 | |
16 | 16 | /data/spark/bin/spark-submit \ |
17 | 17 | --class com.piki_ds.preprocess.MakingLapse \ |
18 | ---master yarn-cluster \ | |
19 | ---conf "spark.yarn.maxAppAttempts=1" \ | |
18 | +--master yarn-client \ | |
20 | 19 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
21 | 20 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
22 | 21 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
mapwithrank.sh
View file @
1e44a60
... | ... | @@ -15,8 +15,7 @@ |
15 | 15 | |
16 | 16 | /data/spark/bin/spark-submit \ |
17 | 17 | --class com.piki_ds.preprocess.MapWithRank \ |
18 | ---master yarn-cluster \ | |
19 | ---conf "spark.yarn.maxAppAttempts=1" \ | |
18 | +--master yarn-client \ | |
20 | 19 | --executor-memory 4000m \ |
21 | 20 | --driver-memory 5000m \ |
22 | 21 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
preprocess.sh
View file @
1e44a60
... | ... | @@ -22,8 +22,7 @@ |
22 | 22 | |
23 | 23 | /data/spark/bin/spark-submit \ |
24 | 24 | --class com.piki_ds.preprocess.$1 \ |
25 | ---master yarn-cluster \ | |
26 | ---conf "spark.yarn.maxAppAttempts=1" \ | |
25 | +--master yarn-client \ | |
27 | 26 | --executor-memory 3000m \ |
28 | 27 | --conf "spark.driver.memory=2g" \ |
29 | 28 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
score.sh
View file @
1e44a60
... | ... | @@ -22,8 +22,7 @@ |
22 | 22 | |
23 | 23 | /data/spark/bin/spark-submit \ |
24 | 24 | --class com.piki_ds.ver1.$1 \ |
25 | ---master yarn-cluster \ | |
26 | ---conf "spark.yarn.maxAppAttempts=1" \ | |
25 | +--master yarn-client \ | |
27 | 26 | --conf "spark.default.parallelism=250" \ |
28 | 27 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
29 | 28 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |