data science team / dsquality

Download as
Browse Code »

Commit ab5b0faaebaa03416c4387bbea91ed72c9a4e071

Authored by 박용규 ago
1 parent 1e44a601b7
Exists in master

change deploy-mode to cluster

Showing 16 changed files with 32 additions and 26 deletions Side-by-side Diff

app/com/piki_ds/preprocess/HourlyCommentCnt.scala View file @ ab5b0fa
... ... @@ -24,8 +24,8 @@
24 24 def getSparkConf= {
25 25 //System.setProperty("SPARK_YARN_MODE", "true")
26 26 val conf = new SparkConf().setAppName("HourlyCommentCnt")
27   - conf.setMaster("yarn-client")
28   - conf.set("master", "yarn-client")
  27 +// conf.setMaster("yarn-client")
  28 +// conf.set("master", "yarn-client")
29 29 conf.set("spark.app.name", "HourlyCommentCnt")
30 30 conf.set("spark.akka.frameSize", "1024")
31 31 }
app/com/piki_ds/preprocess/MakingLapse.scala View file @ ab5b0fa
... ... @@ -19,8 +19,8 @@
19 19 def getSparkConf = {
20 20 //System.setProperty("SPARK_YARN_MODE", "true")
21 21 val conf = new SparkConf().setAppName("MakingLapse")
22   - conf.setMaster("yarn-client")
23   - conf.set("master", "yarn-client")
  22 +// conf.setMaster("yarn-client")
  23 +// conf.set("master", "yarn-client")
24 24 conf.set("spark.app.name", "MakingLapse")
25 25 }
26 26  
app/com/piki_ds/preprocess/MapWithRank.scala View file @ ab5b0fa
... ... @@ -16,8 +16,8 @@
16 16  
17 17 def getSparkConf = {
18 18 val conf = new SparkConf().setAppName("MapWithRank")
19   - conf.setMaster("yarn-client")
20   - conf.set("master", "yarn-client")
  19 +// conf.setMaster("yarn-client")
  20 +// conf.set("master", "yarn-client")
21 21 conf.set("spark.app.name", "MapWithRank")
22 22 }
23 23  
app/com/piki_ds/preprocess/WeeklyECTbyGroup.scala View file @ ab5b0fa
... ... @@ -20,8 +20,8 @@
20 20 def getSparkConf= {
21 21 //System.setProperty("SPARK_YARN_MODE", "true")
22 22 val conf = new SparkConf().setAppName("WeeklyECTbyGroup")
23   - conf.setMaster("yarn-client")
24   - conf.set("master", "yarn-client")
  23 +// conf.setMaster("yarn-client")
  24 +// conf.set("master", "yarn-client")
25 25 conf.set("spark.app.name", "WeeklyECTbyGroup")
26 26 conf.set("spark.akka.frameSize", "1024")
27 27 }
app/com/piki_ds/ver1/CmtScore.scala View file @ ab5b0fa
... ... @@ -16,8 +16,8 @@
16 16 object CmtScore {
17 17 def getSparkConf= {
18 18 val conf = new SparkConf().setAppName("CmtScore")
19   - conf.setMaster("yarn-client")
20   - conf.set("master", "yarn-client")
  19 +// conf.setMaster("yarn-client")
  20 +// conf.set("master", "yarn-client")
21 21 conf.set("spark.app.name", "CmtScore")
22 22 conf.set("spark.akka.frameSize", "1024")
23 23 }
app/com/piki_ds/ver1/ConEfficiencyScore.scala View file @ ab5b0fa
... ... @@ -19,8 +19,8 @@
19 19 object ConEfficiencyScore {
20 20 def getSparkConf= {
21 21 val conf = new SparkConf().setAppName("ConEfficiencyScore")
22   - conf.setMaster("yarn-client")
23   - conf.set("master", "yarn-client")
  22 +// conf.setMaster("yarn-client")
  23 +// conf.set("master", "yarn-client")
24 24 conf.set("spark.app.name", "ConEfficiencyScore")
25 25 conf.set("spark.akka.frameSize", "1024")
26 26 }
app/com/piki_ds/ver1/ConRejectScore.scala View file @ ab5b0fa
... ... @@ -24,8 +24,8 @@
24 24  
25 25 def getSparkConf = {
26 26 val conf = new SparkConf().setAppName("ConRejectScore")
27   - conf.setMaster("yarn-client")
28   - conf.set("master", "yarn-client")
  27 +// conf.setMaster("yarn-client")
  28 +// conf.set("master", "yarn-client")
29 29 conf.set("spark.app.name", "ConRejectScore")
30 30 }
31 31  
app/com/piki_ds/ver1/ContentScore.scala View file @ ab5b0fa
... ... @@ -22,8 +22,8 @@
22 22 def getSparkConf= {
23 23 //System.setProperty("SPARK_YARN_MODE", "true")
24 24 val conf = new SparkConf().setAppName("ContentScore")
25   - conf.setMaster("yarn-client")
26   - conf.set("master", "yarn-client")
  25 +// conf.setMaster("yarn-client")
  26 +// conf.set("master", "yarn-client")
27 27 conf.set("spark.app.name", "ContentScore")
28 28 conf.set("spark.driver.allowMultipleContexts", "true")
29 29 conf.set("spark.akka.frameSize", "1024")
app/com/piki_ds/ver1/DashRelatedScore.scala View file @ ab5b0fa
... ... @@ -25,8 +25,8 @@
25 25 def getSparkConf= {
26 26 //System.setProperty("SPARK_YARN_MODE", "true")
27 27 val conf = new SparkConf().setAppName("DashRelatedScore")
28   - conf.setMaster("yarn-client")
29   - conf.set("master", "yarn-client")
  28 +// conf.setMaster("yarn-client")
  29 +// conf.set("master", "yarn-client")
30 30 conf.set("spark.app.name", "DashRelatedScore")
31 31 conf.set("spark.akka.frameSize", "1024")
32 32 }
app/com/piki_ds/ver1/QualityScore.scala View file @ ab5b0fa
... ... @@ -18,8 +18,8 @@
18 18  
19 19 def getSparkConf= {
20 20 val conf = new SparkConf().setAppName("QualityScore")
21   - conf.setMaster("yarn-client")
22   - conf.set("master", "yarn-client")
  21 +// conf.setMaster("yarn-client")
  22 +// conf.set("master", "yarn-client")
23 23 conf.set("spark.app.name", "QualityScore")
24 24 conf.set("spark.akka.frameSize", "1024")
25 25 }
editor_score.sh View file @ ab5b0fa
... ... @@ -15,7 +15,8 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.ver1.EditorScore \
18   ---master yarn-client \
  18 +--master yarn-cluster \
  19 +--conf "spark.yarn.maxAppAttempts=1" \
19 20 --conf "spark.default.parallelism=250" \
20 21 --conf "spark.driver.memory=2g" \
21 22 --conf "spark.dynamicAllocation.minExecutors=90" \
gghVer2.sh View file @ ab5b0fa
... ... @@ -22,8 +22,9 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class $1 \
25   ---master yarn-client \
  25 +--master yarn-cluster \
26 26 --executor-memory 3000m \
  27 +--conf "spark.yarn.maxAppAttempts=1" \
27 28 --conf "spark.dynamicAllocation.minExecutors=90" \
28 29 --conf "spark.dynamicAllocation.maxExecutors=170" \
29 30 --conf "spark.akka.frameSize=2047" \
makelapse.sh View file @ ab5b0fa
... ... @@ -15,7 +15,8 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.preprocess.MakingLapse \
18   ---master yarn-client \
  18 +--master yarn-cluster \
  19 +--conf "spark.yarn.maxAppAttempts=1" \
19 20 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
20 21 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
21 22 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
mapwithrank.sh View file @ ab5b0fa
... ... @@ -15,9 +15,10 @@
15 15  
16 16 /data/spark/bin/spark-submit \
17 17 --class com.piki_ds.preprocess.MapWithRank \
18   ---master yarn-client \
  18 +--master yarn-cluster \
19 19 --executor-memory 4000m \
20 20 --driver-memory 5000m \
  21 +--conf "spark.yarn.maxAppAttempts=1" \
21 22 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
22 23 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
23 24 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
preprocess.sh View file @ ab5b0fa
... ... @@ -22,7 +22,7 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class com.piki_ds.preprocess.$1 \
25   ---master yarn-client \
  25 +--master yarn-cluster \
26 26 --executor-memory 3000m \
27 27 --conf "spark.driver.memory=2g" \
28 28 --conf "spark.dynamicAllocation.minExecutors=90" \
... ... @@ -32,6 +32,7 @@
32 32 --conf "spark.shuffle.memoryFraction=0.5" \
33 33 --conf "spark.shuffle.manager=hash" \
34 34 --conf "spark.default.parallelism=250" \
  35 +--conf "spark.yarn.maxAppAttempts=1" \
35 36 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
36 37 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
37 38 #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
score.sh View file @ ab5b0fa
... ... @@ -22,7 +22,8 @@
22 22  
23 23 /data/spark/bin/spark-submit \
24 24 --class com.piki_ds.ver1.$1 \
25   ---master yarn-client \
  25 +--master yarn-cluster \
  26 +--conf "spark.yarn.maxAppAttempts=1" \
26 27 --conf "spark.default.parallelism=250" \
27 28 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1
28 29 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1