Commit eb25d60633281ed71962a572e490e6e777b7b01a

Authored by Joanne ago
1 parent 783fe3a33d
Exists in master

editor score

Showing 1 changed file with 4 additions and 2 deletions Side-by-side Diff

app/com/piki_ds/ver1/EditorScore.scala View file @ eb25d60
... ... @@ -38,7 +38,7 @@
38 38 list.filter(x=>x.isDirectory && (!isParted || (isParted && hdfs.exists(x.getPath.suffix("/_SUCCESS"))))).maxBy(x=>x.getModificationTime)
39 39 }
40 40  
41   - def followGetter(sQLContext: SQLContext, dateKey:String) = {
  41 + def followGetter(sQLContext: SQLContext, dateKey:String, fileSave:Boolean = true) = {
42 42 val fromExisting = sQLContext.read.format("json").load(recentlyUpdatedPath("/preprocess/followInfo",true,fs).getPath.toString)
43 43 val fromUpdated = getDashTable(sQLContext, "EDITOR_FOLLOW", dateKey)
44 44 val unionFrom = fromExisting.unionAll(fromUpdated)
... ... @@ -46,7 +46,9 @@
46 46 Map("cdate" -> x.getAs[String](0).toString, "follow_cnt" -> x.getAs[Long](1).toString, "uid" -> x.getAs[Long](3).toString)
47 47 })
48 48 val follow_info = intoMap.groupBy(x=>x("uid")).map(x=>(x._1, x._2.map(a=>a("follow_cnt").toLong).sum))
49   - unionFrom.write.format("json").save(s"hdfs://pikinn/preprocess/followInfo/$dateKey")
  49 + if (fileSave) {
  50 + unionFrom.write.format("json").save(s"hdfs://pikinn/preprocess/followInfo/$dateKey")
  51 + }
50 52 follow_info
51 53 }
52 54