Commit 417d410163b302f8683451a35344751534dd263b

Authored by Joanne ago
1 parent ddba312d26
Exists in master

cid by status

Showing 1 changed file with 18 additions and 0 deletions Side-by-side Diff

app/com/piki_ds/preprocess/CidValidation.scala View file @ 417d410
... ... @@ -28,5 +28,23 @@
28 28  
29 29 })
30 30 }
  31 +
  32 + def getCidByStatus(sQLContext: SQLContext, filterStatus:Array[String]) = {
  33 + import org.apache.spark.sql.functions._
  34 + val whereStr = s"udate id not null and title is not null and" +
  35 + s" contents_type in ('ALBUM', 'ALBUM.A', 'CHST', 'CHST.A','TOON','TOON.A') and " +
  36 + s"status in (${filterStatus.map(x=>s"'$x'").mkString(",")})"
  37 + val mgc = getDBDump(sQLContext,"MG_CONTENTS").where(whereStr)
  38 + val mgContents = mgc.select(mgc("contents_id"),mgc("status"), unix_timestamp(mgc("udate")))
  39 + mgContents.map(x=>{
  40 + val ts = x.getAs[Long]("unixtimestamp(udate,yyyy-MM-dd HH:mm:ss)")
  41 + val status = if (x.getAs[String]("status").equals("ACTV")) 1 else 0
  42 + (x.getAs[Long]("contents_id"), (status, ts))
  43 + }).reduceByKey((a,b) => {
  44 + import math.{min,max}
  45 + (max(a._1,b._1), min(a._2,b._2))
  46 +
  47 + })
  48 + }
31 49 }