Commit ab5b0faaebaa03416c4387bbea91ed72c9a4e071
1 parent
1e44a601b7
Exists in
master
change deploy-mode to cluster
Showing 16 changed files with 32 additions and 26 deletions Side-by-side Diff
- app/com/piki_ds/preprocess/HourlyCommentCnt.scala
- app/com/piki_ds/preprocess/MakingLapse.scala
- app/com/piki_ds/preprocess/MapWithRank.scala
- app/com/piki_ds/preprocess/WeeklyECTbyGroup.scala
- app/com/piki_ds/ver1/CmtScore.scala
- app/com/piki_ds/ver1/ConEfficiencyScore.scala
- app/com/piki_ds/ver1/ConRejectScore.scala
- app/com/piki_ds/ver1/ContentScore.scala
- app/com/piki_ds/ver1/DashRelatedScore.scala
- app/com/piki_ds/ver1/QualityScore.scala
- editor_score.sh
- gghVer2.sh
- makelapse.sh
- mapwithrank.sh
- preprocess.sh
- score.sh
app/com/piki_ds/preprocess/HourlyCommentCnt.scala
View file @
ab5b0fa
... | ... | @@ -24,8 +24,8 @@ |
24 | 24 | def getSparkConf= { |
25 | 25 | //System.setProperty("SPARK_YARN_MODE", "true") |
26 | 26 | val conf = new SparkConf().setAppName("HourlyCommentCnt") |
27 | - conf.setMaster("yarn-client") | |
28 | - conf.set("master", "yarn-client") | |
27 | +// conf.setMaster("yarn-client") | |
28 | +// conf.set("master", "yarn-client") | |
29 | 29 | conf.set("spark.app.name", "HourlyCommentCnt") |
30 | 30 | conf.set("spark.akka.frameSize", "1024") |
31 | 31 | } |
app/com/piki_ds/preprocess/MakingLapse.scala
View file @
ab5b0fa
... | ... | @@ -19,8 +19,8 @@ |
19 | 19 | def getSparkConf = { |
20 | 20 | //System.setProperty("SPARK_YARN_MODE", "true") |
21 | 21 | val conf = new SparkConf().setAppName("MakingLapse") |
22 | - conf.setMaster("yarn-client") | |
23 | - conf.set("master", "yarn-client") | |
22 | +// conf.setMaster("yarn-client") | |
23 | +// conf.set("master", "yarn-client") | |
24 | 24 | conf.set("spark.app.name", "MakingLapse") |
25 | 25 | } |
26 | 26 |
app/com/piki_ds/preprocess/MapWithRank.scala
View file @
ab5b0fa
... | ... | @@ -16,8 +16,8 @@ |
16 | 16 | |
17 | 17 | def getSparkConf = { |
18 | 18 | val conf = new SparkConf().setAppName("MapWithRank") |
19 | - conf.setMaster("yarn-client") | |
20 | - conf.set("master", "yarn-client") | |
19 | +// conf.setMaster("yarn-client") | |
20 | +// conf.set("master", "yarn-client") | |
21 | 21 | conf.set("spark.app.name", "MapWithRank") |
22 | 22 | } |
23 | 23 |
app/com/piki_ds/preprocess/WeeklyECTbyGroup.scala
View file @
ab5b0fa
... | ... | @@ -20,8 +20,8 @@ |
20 | 20 | def getSparkConf= { |
21 | 21 | //System.setProperty("SPARK_YARN_MODE", "true") |
22 | 22 | val conf = new SparkConf().setAppName("WeeklyECTbyGroup") |
23 | - conf.setMaster("yarn-client") | |
24 | - conf.set("master", "yarn-client") | |
23 | +// conf.setMaster("yarn-client") | |
24 | +// conf.set("master", "yarn-client") | |
25 | 25 | conf.set("spark.app.name", "WeeklyECTbyGroup") |
26 | 26 | conf.set("spark.akka.frameSize", "1024") |
27 | 27 | } |
app/com/piki_ds/ver1/CmtScore.scala
View file @
ab5b0fa
... | ... | @@ -16,8 +16,8 @@ |
16 | 16 | object CmtScore { |
17 | 17 | def getSparkConf= { |
18 | 18 | val conf = new SparkConf().setAppName("CmtScore") |
19 | - conf.setMaster("yarn-client") | |
20 | - conf.set("master", "yarn-client") | |
19 | +// conf.setMaster("yarn-client") | |
20 | +// conf.set("master", "yarn-client") | |
21 | 21 | conf.set("spark.app.name", "CmtScore") |
22 | 22 | conf.set("spark.akka.frameSize", "1024") |
23 | 23 | } |
app/com/piki_ds/ver1/ConEfficiencyScore.scala
View file @
ab5b0fa
... | ... | @@ -19,8 +19,8 @@ |
19 | 19 | object ConEfficiencyScore { |
20 | 20 | def getSparkConf= { |
21 | 21 | val conf = new SparkConf().setAppName("ConEfficiencyScore") |
22 | - conf.setMaster("yarn-client") | |
23 | - conf.set("master", "yarn-client") | |
22 | +// conf.setMaster("yarn-client") | |
23 | +// conf.set("master", "yarn-client") | |
24 | 24 | conf.set("spark.app.name", "ConEfficiencyScore") |
25 | 25 | conf.set("spark.akka.frameSize", "1024") |
26 | 26 | } |
app/com/piki_ds/ver1/ConRejectScore.scala
View file @
ab5b0fa
... | ... | @@ -24,8 +24,8 @@ |
24 | 24 | |
25 | 25 | def getSparkConf = { |
26 | 26 | val conf = new SparkConf().setAppName("ConRejectScore") |
27 | - conf.setMaster("yarn-client") | |
28 | - conf.set("master", "yarn-client") | |
27 | +// conf.setMaster("yarn-client") | |
28 | +// conf.set("master", "yarn-client") | |
29 | 29 | conf.set("spark.app.name", "ConRejectScore") |
30 | 30 | } |
31 | 31 |
app/com/piki_ds/ver1/ContentScore.scala
View file @
ab5b0fa
... | ... | @@ -22,8 +22,8 @@ |
22 | 22 | def getSparkConf= { |
23 | 23 | //System.setProperty("SPARK_YARN_MODE", "true") |
24 | 24 | val conf = new SparkConf().setAppName("ContentScore") |
25 | - conf.setMaster("yarn-client") | |
26 | - conf.set("master", "yarn-client") | |
25 | +// conf.setMaster("yarn-client") | |
26 | +// conf.set("master", "yarn-client") | |
27 | 27 | conf.set("spark.app.name", "ContentScore") |
28 | 28 | conf.set("spark.driver.allowMultipleContexts", "true") |
29 | 29 | conf.set("spark.akka.frameSize", "1024") |
app/com/piki_ds/ver1/DashRelatedScore.scala
View file @
ab5b0fa
... | ... | @@ -25,8 +25,8 @@ |
25 | 25 | def getSparkConf= { |
26 | 26 | //System.setProperty("SPARK_YARN_MODE", "true") |
27 | 27 | val conf = new SparkConf().setAppName("DashRelatedScore") |
28 | - conf.setMaster("yarn-client") | |
29 | - conf.set("master", "yarn-client") | |
28 | +// conf.setMaster("yarn-client") | |
29 | +// conf.set("master", "yarn-client") | |
30 | 30 | conf.set("spark.app.name", "DashRelatedScore") |
31 | 31 | conf.set("spark.akka.frameSize", "1024") |
32 | 32 | } |
app/com/piki_ds/ver1/QualityScore.scala
View file @
ab5b0fa
... | ... | @@ -18,8 +18,8 @@ |
18 | 18 | |
19 | 19 | def getSparkConf= { |
20 | 20 | val conf = new SparkConf().setAppName("QualityScore") |
21 | - conf.setMaster("yarn-client") | |
22 | - conf.set("master", "yarn-client") | |
21 | +// conf.setMaster("yarn-client") | |
22 | +// conf.set("master", "yarn-client") | |
23 | 23 | conf.set("spark.app.name", "QualityScore") |
24 | 24 | conf.set("spark.akka.frameSize", "1024") |
25 | 25 | } |
editor_score.sh
View file @
ab5b0fa
... | ... | @@ -15,7 +15,8 @@ |
15 | 15 | |
16 | 16 | /data/spark/bin/spark-submit \ |
17 | 17 | --class com.piki_ds.ver1.EditorScore \ |
18 | ---master yarn-client \ | |
18 | +--master yarn-cluster \ | |
19 | +--conf "spark.yarn.maxAppAttempts=1" \ | |
19 | 20 | --conf "spark.default.parallelism=250" \ |
20 | 21 | --conf "spark.driver.memory=2g" \ |
21 | 22 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
gghVer2.sh
View file @
ab5b0fa
... | ... | @@ -22,8 +22,9 @@ |
22 | 22 | |
23 | 23 | /data/spark/bin/spark-submit \ |
24 | 24 | --class $1 \ |
25 | ---master yarn-client \ | |
25 | +--master yarn-cluster \ | |
26 | 26 | --executor-memory 3000m \ |
27 | +--conf "spark.yarn.maxAppAttempts=1" \ | |
27 | 28 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
28 | 29 | --conf "spark.dynamicAllocation.maxExecutors=170" \ |
29 | 30 | --conf "spark.akka.frameSize=2047" \ |
makelapse.sh
View file @
ab5b0fa
... | ... | @@ -15,7 +15,8 @@ |
15 | 15 | |
16 | 16 | /data/spark/bin/spark-submit \ |
17 | 17 | --class com.piki_ds.preprocess.MakingLapse \ |
18 | ---master yarn-client \ | |
18 | +--master yarn-cluster \ | |
19 | +--conf "spark.yarn.maxAppAttempts=1" \ | |
19 | 20 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
20 | 21 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
21 | 22 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
mapwithrank.sh
View file @
ab5b0fa
... | ... | @@ -15,9 +15,10 @@ |
15 | 15 | |
16 | 16 | /data/spark/bin/spark-submit \ |
17 | 17 | --class com.piki_ds.preprocess.MapWithRank \ |
18 | ---master yarn-client \ | |
18 | +--master yarn-cluster \ | |
19 | 19 | --executor-memory 4000m \ |
20 | 20 | --driver-memory 5000m \ |
21 | +--conf "spark.yarn.maxAppAttempts=1" \ | |
21 | 22 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
22 | 23 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
23 | 24 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
preprocess.sh
View file @
ab5b0fa
... | ... | @@ -22,7 +22,7 @@ |
22 | 22 | |
23 | 23 | /data/spark/bin/spark-submit \ |
24 | 24 | --class com.piki_ds.preprocess.$1 \ |
25 | ---master yarn-client \ | |
25 | +--master yarn-cluster \ | |
26 | 26 | --executor-memory 3000m \ |
27 | 27 | --conf "spark.driver.memory=2g" \ |
28 | 28 | --conf "spark.dynamicAllocation.minExecutors=90" \ |
... | ... | @@ -32,6 +32,7 @@ |
32 | 32 | --conf "spark.shuffle.memoryFraction=0.5" \ |
33 | 33 | --conf "spark.shuffle.manager=hash" \ |
34 | 34 | --conf "spark.default.parallelism=250" \ |
35 | +--conf "spark.yarn.maxAppAttempts=1" \ | |
35 | 36 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
36 | 37 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
37 | 38 | #target/scala-2.11/dsmakingscore_2.11-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
score.sh
View file @
ab5b0fa
... | ... | @@ -22,7 +22,8 @@ |
22 | 22 | |
23 | 23 | /data/spark/bin/spark-submit \ |
24 | 24 | --class com.piki_ds.ver1.$1 \ |
25 | ---master yarn-client \ | |
25 | +--master yarn-cluster \ | |
26 | +--conf "spark.yarn.maxAppAttempts=1" \ | |
26 | 27 | --conf "spark.default.parallelism=250" \ |
27 | 28 | $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |
28 | 29 | #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1 |