data science team / dsquality

Authored by Joanne about 6 years ago

Exists in master

Showing 2 changed files with 6 additions and 9 deletions Side-by-side Diff

...	...	@@ -31,7 +31,7 @@
31	31
32	32	def getCidByStatus(sQLContext: SQLContext, filterStatus:Array[String]) = {
33	33	import org.apache.spark.sql.functions._
34		- val whereStr = s"udate is not null and id is not null and title is not null and" +
	34	+ val whereStr = s"udate is not null and contents_id is not null and title is not null and" +
35	35	s" contents_type in ('ALBUM', 'ALBUM.A', 'CHST', 'CHST.A','TOON','TOON.A') and " +
36	36	s"status in (${filterStatus.map(x=>s"'$x'").mkString(",")})"
37	37	val mgc = getDBDump(sQLContext,"MG_CONTENTS").where(whereStr)

1	1	package com.piki_ds.ver1
2	2
3		-import org.apache.hadoop.fs.{Path, FileStatus, FileSystem}
4		-import org.jblas.MatrixFunctions
5		-
6	3	import org.apache.spark.rdd.RDD
	4	+import org.apache.hadoop.fs.{Path, FileStatus, FileSystem}
7	5	import org.apache.spark.sql.SQLContext
8	6	import org.apache.spark.{SparkConf, SparkContext}
	7	+import org.jblas.MatrixFunctions
9	8
10		-import com.piki_ds.utils.DateTimeEtc.intoYesterdayMN
	9	+import com.piki_ds.utils.DateTimeEtc.getDateKey
11	10	import com.piki_ds.utils.GetTextFile.getDashTable
12	11	import com.piki_ds.utils.SqlContextConf.readPartialTable
13	12	import com.piki_ds.utils.TempScoreSaveLoad
...	...	@@ -40,7 +39,6 @@
40	39	}
41	40
42	41	def followGetter(sQLContext: SQLContext, dateKey:String) = {
43		-
44	42	val fromExisting = sQLContext.read.format("json").load(recentlyUpdatedPath("/preprocess/followInfo",true,fs).getPath.toString)
45	43	val fromUpdated = getDashTable(sQLContext, "EDITOR_FOLLOW", dateKey)
46	44	val unionFrom = fromExisting.unionAll(fromUpdated)
47	45
...	...	@@ -92,10 +90,9 @@
92	90	}
93	91
94	92	def main(args:Array[String]) {
95		- //현재시점 고정해놓기 [계산시점]Long = 1435261946000
96	93	val nowTS: Long = System.currentTimeMillis
97		- val yesterdayTuple = intoYesterdayMN(nowTS)
98		- val dateKey = yesterdayTuple._2.replaceAll("[^0-9]", "").take(8)
	94	+ val dateKey = getDateKey(nowTS)
	95	+
99	96	//에디터 인기 점수 구하기
100	97	val fromDB = editorDB(sqlContext, dateKey)
101	98