Commit 8da4773a0a84e660a8c82049350020afc00c3b45
1 parent
c22ba02971
Exists in
master
cid validation
Showing 2 changed files with 10 additions and 6 deletions Side-by-side Diff
app/com/piki_ds/preprocess/CidValidation.scala
View file @
8da4773
| ... | ... | @@ -13,7 +13,11 @@ |
| 13 | 13 | |
| 14 | 14 | def getCidByStatus(sQLContext: SQLContext, filterStatus:Array[String]) = { |
| 15 | 15 | import org.apache.spark.sql.functions._ |
| 16 | - val mgc = getDashDump(sQLContext,"MG_CONTENTS").where(s"status in (${filterStatus.map(x=>s"'$x'").mkString(",")})") | |
| 16 | + val whereStr = s"udate is not null and title is not null and" + | |
| 17 | + s" contents_type in ('ALBUM', 'ALBUM.A', 'CHST', 'CHST.A') and " + | |
| 18 | + s"contents_type in ('ALBUM', 'ALBUM.A', 'CHST', 'CHST.A') and " + | |
| 19 | + s"status in (${filterStatus.map(x=>s"'$x'").mkString(",")})" | |
| 20 | + val mgc = getDashDump(sQLContext,"MG_CONTENTS").where(whereStr) | |
| 17 | 21 | val mgContents = mgc.select(mgc("contents_id"),mgc("status"), unix_timestamp(mgc("udate"))) |
| 18 | 22 | mgContents.map(x=>{ |
| 19 | 23 | val ts = x.getAs[Long]("unixtimestamp(udate,yyyy-MM-dd HH:mm:ss)") |
app/com/piki_ds/ver1/DashRelatedScore.scala
View file @
8da4773
| ... | ... | @@ -2,17 +2,17 @@ |
| 2 | 2 | |
| 3 | 3 | import org.apache.spark.rdd.RDD |
| 4 | 4 | import org.apache.spark.sql.SQLContext |
| 5 | -import org.apache.spark.sql.functions._ | |
| 5 | +import org.apache.spark.sql.functions.udf | |
| 6 | 6 | import org.apache.spark.{SparkConf, SparkContext} |
| 7 | 7 | import org.slf4j.LoggerFactory |
| 8 | 8 | |
| 9 | -import com.piki_ds.ver1.ConClickScore.clickScoreCalc | |
| 10 | -import com.piki_ds.ver1.ConDwellScore.dwellScoreCalc | |
| 11 | -import com.piki_ds.ver1.ConPostActionScore.pAScoreCalc | |
| 12 | -import com.piki_ds.preprocess.CidValidation.getCidByStatus | |
| 13 | 9 | import com.piki_ds.utils.DateTimeEtc.{intoNDaysDT, getDateKey} |
| 14 | 10 | import com.piki_ds.utils.GetTextFile.getDashTable |
| 15 | 11 | import com.piki_ds.utils.TempScoreSaveLoad.scoreSave |
| 12 | +import com.piki_ds.preprocess.CidValidation.getCidByStatus | |
| 13 | +import com.piki_ds.ver1.ConClickScore.clickScoreCalc | |
| 14 | +import com.piki_ds.ver1.ConDwellScore.dwellScoreCalc | |
| 15 | +import com.piki_ds.ver1.ConPostActionScore.pAScoreCalc | |
| 16 | 16 | |
| 17 | 17 | /** |
| 18 | 18 | * Created by jungwon on 12/17/15. |