Commit 8da4773a0a84e660a8c82049350020afc00c3b45

Authored by Joanne ago
1 parent c22ba02971
Exists in master

cid validation

Showing 2 changed files with 10 additions and 6 deletions Side-by-side Diff

app/com/piki_ds/preprocess/CidValidation.scala View file @ 8da4773
... ... @@ -13,7 +13,11 @@
13 13  
14 14 def getCidByStatus(sQLContext: SQLContext, filterStatus:Array[String]) = {
15 15 import org.apache.spark.sql.functions._
16   - val mgc = getDashDump(sQLContext,"MG_CONTENTS").where(s"status in (${filterStatus.map(x=>s"'$x'").mkString(",")})")
  16 + val whereStr = s"udate is not null and title is not null and" +
  17 + s" contents_type in ('ALBUM', 'ALBUM.A', 'CHST', 'CHST.A') and " +
  18 + s"contents_type in ('ALBUM', 'ALBUM.A', 'CHST', 'CHST.A') and " +
  19 + s"status in (${filterStatus.map(x=>s"'$x'").mkString(",")})"
  20 + val mgc = getDashDump(sQLContext,"MG_CONTENTS").where(whereStr)
17 21 val mgContents = mgc.select(mgc("contents_id"),mgc("status"), unix_timestamp(mgc("udate")))
18 22 mgContents.map(x=>{
19 23 val ts = x.getAs[Long]("unixtimestamp(udate,yyyy-MM-dd HH:mm:ss)")
app/com/piki_ds/ver1/DashRelatedScore.scala View file @ 8da4773
... ... @@ -2,17 +2,17 @@
2 2  
3 3 import org.apache.spark.rdd.RDD
4 4 import org.apache.spark.sql.SQLContext
5   -import org.apache.spark.sql.functions._
  5 +import org.apache.spark.sql.functions.udf
6 6 import org.apache.spark.{SparkConf, SparkContext}
7 7 import org.slf4j.LoggerFactory
8 8  
9   -import com.piki_ds.ver1.ConClickScore.clickScoreCalc
10   -import com.piki_ds.ver1.ConDwellScore.dwellScoreCalc
11   -import com.piki_ds.ver1.ConPostActionScore.pAScoreCalc
12   -import com.piki_ds.preprocess.CidValidation.getCidByStatus
13 9 import com.piki_ds.utils.DateTimeEtc.{intoNDaysDT, getDateKey}
14 10 import com.piki_ds.utils.GetTextFile.getDashTable
15 11 import com.piki_ds.utils.TempScoreSaveLoad.scoreSave
  12 +import com.piki_ds.preprocess.CidValidation.getCidByStatus
  13 +import com.piki_ds.ver1.ConClickScore.clickScoreCalc
  14 +import com.piki_ds.ver1.ConDwellScore.dwellScoreCalc
  15 +import com.piki_ds.ver1.ConPostActionScore.pAScoreCalc
16 16  
17 17 /**
18 18 * Created by jungwon on 12/17/15.