Commit 1f8e2feceb3536e12e0f3969390f68755926cdeb

Authored by Joanne ago
1 parent 8d33cc0e41
Exists in master

filter out 0 consume time

Showing 1 changed file with 2 additions and 2 deletions Side-by-side Diff

app/com/piki_ds/preprocess/WeeklyECTbyGroup.scala View file @ 1f8e2fe
... ... @@ -116,11 +116,11 @@
116 116 val cardSize = getCardSize()
117 117 cardSize.saveAsObjectFile(s"hdfs://pikinn/preprocess/cidAndCardSize/$dateKey")
118 118  
119   - val ctbyCardSize = weeklyCT.join(cardSize).groupBy(x=>x._2._2).map(x=>{
  119 + val ctbyCardSize: RDD[(Int, Long)] = weeklyCT.join(cardSize).groupBy(x=>x._2._2).map(x=>{
120 120 val consumeTime = x._2.map(_._2._1).sum
121 121 (x._1,consumeTime)
122 122 })
123   - ctbyCardSize.saveAsObjectFile(s"hdfs://pikinn/preprocess/ctByCardSize/$dateKey")
  123 + ctbyCardSize.filter(_._2 != 0L).saveAsObjectFile(s"hdfs://pikinn/preprocess/ctByCardSize/$dateKey")
124 124  
125 125 val cidwithUid = cidWithUid()
126 126 cidwithUid.saveAsObjectFile(s"hdfs://pikinn/preprocess/cidWithUid/$dateKey")