Commit 3a990d5ee56edfc4bf4a153d79b30dc4a54a6c02

Authored by evan ago
1 parent 5eb0e11740
Exists in master

sh memory 3000 and fs check change

Showing 3 changed files with 15 additions and 3 deletions Side-by-side Diff

app/com/piki_ds/ver2ggh/expConTime.scala View file @ 3a990d5
... ... @@ -2,6 +2,9 @@
2 2 * Created by Evan on 2016. 5. 10..
3 3 */
4 4 package com.piki_ds.ver2ggh
  5 +
  6 +import java.util.Date
  7 +
5 8 import org.apache.spark.SparkContext
6 9 import org.apache.spark.rdd.RDD
7 10 import org.apache.spark.sql.{DataFrame, SaveMode, SQLContext}
8 11  
... ... @@ -174,9 +177,11 @@
174 177 }
175 178  
176 179 def main(args: Array[String]): Unit = {
177   -
  180 + val format = new java.text.SimpleDateFormat("yyyyMMdd")
  181 + val currentTime = new Date()
  182 + val day_delm = 24 * 1000 * 60 * 60L
178 183 // 로그 파스
179   - val saveDay = if(args.length == 1) args(0) else "20160518"
  184 + val saveDay = if(args.length == 1) args(0) else format.format(currentTime.getTime - day_delm)
180 185 val df1 = getLogParse(saveDay)
181 186  
182 187 // 로그를 (오픈, 소비) 정보와 (소비, 오픈) 정보 둘로 나눔
app/com/piki_ds/ver2ggh/gghScore.scala View file @ 3a990d5
... ... @@ -44,7 +44,7 @@
44 44  
45 45 dateSet.map{x =>
46 46 val addrs = s"hdfs://pikinn/user/evan/Features/table=expConTime/dt=${x}"
47   - val out = if(!fs.exists((new Path(addrs)))) {x} else "null"
  47 + val out = if(fs.listStatus((new Path(addrs))).size < 100) {x} else "null"
48 48 out
49 49 }.filter(x => x != "null").map(x => ver2ggh.expConTime.main(Array(s"${x}")))
50 50  
... ... @@ -23,6 +23,13 @@
23 23 /data/spark/bin/spark-submit \
24 24 --class $1 \
25 25 --master yarn-client \
  26 +--executor-memory 3000m \
  27 +--conf "spark.dynamicAllocation.minExecutors=90" \
  28 +--conf "spark.dynamicAllocation.maxExecutors=170" \
  29 +--conf "spark.akka.frameSize=2047" \
  30 +--conf "spark.network.timeout=120000" \
  31 +--conf "spark.shuffle.memoryFraction=0.5" \
  32 +--conf "spark.shuffle.manager=hash" \
26 33 --conf "spark.default.parallelism=250" \
27 34 $BASEDIR/target/scala-2.11/dsquality-assembly-0.1.0-SNAPSHOT.jar $2 >> $LOG 2>&1
28 35 #target/scala-2.11/dsmakingscore-assembly-0.1.0-SNAPSHOT.jar >> $LOG 2>&1