Commit 8230469489a05b64a8daebf7e989c49499e1f22c

Authored by Joanne ago
1 parent 1f8e2feceb
Exists in master

nonzero

Showing 1 changed file with 1 additions and 1 deletions Side-by-side Diff

app/com/piki_ds/preprocess/MapWithRank.scala View file @ 8230469
... ... @@ -190,7 +190,7 @@
190 190 val log = getLog(sc, doi)
191 191 val joinedLog = joinLogWithLapse(log, getLapse(sc, doi, "CONSUME"))
192 192 val mapped = mapActions(joinedLog, countUser(log, 10000))
193   - val ecWithRank: RDD[(String, (Int, String), (Long, Long))] = mapRanges(mapped)
  193 + val ecWithRank: RDD[(String, (Int, String), (Long, Long))] = mapRanges(mapped).filter(x=>x._3._1 != 0L && x._3._2 != 0L)
194 194  
195 195 val csvMapRank = ecWithRank.map(x=>s"${x._1},${x._2._1},${x._2._2},${x._3._1},${x._3._2}")
196 196 csvMapRank.saveAsTextFile(s"hdfs://pikinn/preprocess/ecWithRank/$doi/")