data science team / dsquality

Download as
Browse Code »

Commit 1e44a601b71039d341ad08312378001cb7b21504

Authored by 박용규 ago
1 parent e3f1f7c03f
Exists in master

deploy-mode revert

Showing 6 changed files with 6 additions and 12 deletions Side-by-side Diff

editor_score.sh View file @ 1e44a60
... ... @@ -15,8 +15,7 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.ver1.EditorScore \
18   ---master yarn-cluster \
19   ---conf "spark.yarn.maxAppAttempts=1" \
  18 +--master yarn-client \
20 19 --conf "spark.default.parallelism=250" \
21 20 --conf "spark.driver.memory=2g" \
22 21 --conf "spark.dynamicAllocation.minExecutors=90" \
gghVer2.sh View file @ 1e44a60
... ... @@ -22,8 +22,7 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class $1 \
25   ---master yarn-cluster \
26   ---conf "spark.yarn.maxAppAttempts=1" \
  25 +--master yarn-client \
27 26 --executor-memory 3000m \
28 27 --conf "spark.dynamicAllocation.minExecutors=90" \
29 28 --conf "spark.dynamicAllocation.maxExecutors=170" \
makelapse.sh View file @ 1e44a60
... ... @@ -15,8 +15,7 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.preprocess.MakingLapse \
18   ---master yarn-cluster \
19   ---conf "spark.yarn.maxAppAttempts=1" \
  18 +--master yarn-client \
20 19 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
21 20 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
22 21 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
mapwithrank.sh View file @ 1e44a60
... ... @@ -15,8 +15,7 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.preprocess.MapWithRank \
18   ---master yarn-cluster \
19   ---conf "spark.yarn.maxAppAttempts=1" \
  18 +--master yarn-client \
20 19 --executor-memory 4000m \
21 20 --driver-memory 5000m \
22 21 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
preprocess.sh View file @ 1e44a60
... ... @@ -22,8 +22,7 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class com.piki_ds.preprocess.$1 \
25   ---master yarn-cluster \
26   ---conf "spark.yarn.maxAppAttempts=1" \
  25 +--master yarn-client \
27 26 --executor-memory 3000m \
28 27 --conf "spark.driver.memory=2g" \
29 28 --conf "spark.dynamicAllocation.minExecutors=90" \
score.sh View file @ 1e44a60
... ... @@ -22,8 +22,7 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class com.piki_ds.ver1.$1 \
25   ---master yarn-cluster \
26   ---conf "spark.yarn.maxAppAttempts=1" \
  25 +--master yarn-client \
27 26 --conf "spark.default.parallelism=250" \
28 27 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
29 28 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1