Commit cf5d677136fea152b1d5b0ccf2c6ef0e61cee9cb

Authored by Joanne ago
1 parent 332b71d3c5
Exists in master

insert debug

Showing 1 changed file with 7 additions and 7 deletions Side-by-side Diff

app/com/piki_ds/ver1/QualityScore.scala View file @ cf5d677
... ... @@ -50,11 +50,11 @@
50 50  
51 51 def combineScores(content:RDD[(Int,Int)], comment:RDD[(Int,Int)], editor:RDD[(Int,Int)],
52 52 param:Map[String,Double]): RDD[(String, Long)] = {
53   - val combine = content.fullOuterJoin(comment).map(x=>{
54   - (x._1, (x._2._1.getOrElse(120),x._2._2.getOrElse(1)))
55   - }).fullOuterJoin(editor).map(x=>{
56   - val b: (Int, (Option[(Int, Int)], Option[Int])) = x
57   - (x._1, (x._2._1.getOrElse((110,112)),x._2._2.getOrElse(121)))
  53 + val combine = content.leftOuterJoin(comment).map(x=>{
  54 + (x._1, (x._2._1,x._2._2.getOrElse(1)))
  55 + }).leftOuterJoin(editor).map(x=>{
  56 + val b = x
  57 + (x._1, ((x._2._1._1, x._2._1._2), x._2._2.getOrElse(121)))
58 58 }).map(x=> {
59 59 (x._1, (x._2._1._1.toDouble, x._2._1._2.toDouble, x._2._2.toDouble))
60 60 }).map(x=>{
61 61  
... ... @@ -69,10 +69,10 @@
69 69 val doi = getDateKey(nowTS)
70 70  
71 71 val content: RDD[(Int, Int)] = scoreLoad(sc,doi,"content","all")
72   - val comment: RDD[(Int, Int)] = scoreLoad(sc,doi,"comment","")
  72 + val comment: RDD[(Int, Int)] = scoreLoad(sc,doi,"comment","").groupByKey().map(x=>(x._1,0))
73 73 val editor: RDD[(Int, Int)] = scoreLoad(sc,doi,"editor","")
74 74  
75   - val param = Map("content"->0.5D,"comment"-> 0.30D,"editor"->0.20D)
  75 + val param = Map("content"->0.5D,"comment"-> 0.0D,"editor"->0.20D)
76 76  
77 77 val finalScore: RDD[(Int, Long)] = combineScores(content,comment,editor,param).map(x=>(x._1.toInt,x._2.toLong)).filter(_._1 != 0)
78 78 scoreSave(doi,"quality","",finalScore.map(x=>(x._1.toString,x._2)),1)