Commit 8230469489a05b64a8daebf7e989c49499e1f22c
1 parent
1f8e2feceb
Exists in
master
nonzero
Showing 1 changed file with 1 additions and 1 deletions Side-by-side Diff
app/com/piki_ds/preprocess/MapWithRank.scala
View file @
8230469
| ... | ... | @@ -190,7 +190,7 @@ |
| 190 | 190 | val log = getLog(sc, doi) |
| 191 | 191 | val joinedLog = joinLogWithLapse(log, getLapse(sc, doi, "CONSUME")) |
| 192 | 192 | val mapped = mapActions(joinedLog, countUser(log, 10000)) |
| 193 | - val ecWithRank: RDD[(String, (Int, String), (Long, Long))] = mapRanges(mapped) | |
| 193 | + val ecWithRank: RDD[(String, (Int, String), (Long, Long))] = mapRanges(mapped).filter(x=>x._3._1 != 0L && x._3._2 != 0L) | |
| 194 | 194 | |
| 195 | 195 | val csvMapRank = ecWithRank.map(x=>s"${x._1},${x._2._1},${x._2._2},${x._3._1},${x._3._2}") |
| 196 | 196 | csvMapRank.saveAsTextFile(s"hdfs://pikinn/preprocess/ecWithRank/$doi/") |