Commit 7393595ae89530e18e8b51ee9a08b3f34aa0ff22

Authored by Joanne ago
1 parent bec43ca4ca
Exists in master

error in sql

Showing 2 changed files with 6 additions and 9 deletions Side-by-side Diff

app/com/piki_ds/preprocess/CidValidation.scala View file @ 7393595
... ... @@ -31,7 +31,7 @@
31 31  
32 32 def getCidByStatus(sQLContext: SQLContext, filterStatus:Array[String]) = {
33 33 import org.apache.spark.sql.functions._
34   - val whereStr = s"udate is not null and id is not null and title is not null and" +
  34 + val whereStr = s"udate is not null and contents_id is not null and title is not null and" +
35 35 s" contents_type in ('ALBUM', 'ALBUM.A', 'CHST', 'CHST.A','TOON','TOON.A') and " +
36 36 s"status in (${filterStatus.map(x=>s"'$x'").mkString(",")})"
37 37 val mgc = getDBDump(sQLContext,"MG_CONTENTS").where(whereStr)
app/com/piki_ds/ver1/EditorScore.scala View file @ 7393595
1 1 package com.piki_ds.ver1
2 2  
3   -import org.apache.hadoop.fs.{Path, FileStatus, FileSystem}
4   -import org.jblas.MatrixFunctions
5   -
6 3 import org.apache.spark.rdd.RDD
  4 +import org.apache.hadoop.fs.{Path, FileStatus, FileSystem}
7 5 import org.apache.spark.sql.SQLContext
8 6 import org.apache.spark.{SparkConf, SparkContext}
  7 +import org.jblas.MatrixFunctions
9 8  
10   -import com.piki_ds.utils.DateTimeEtc.intoYesterdayMN
  9 +import com.piki_ds.utils.DateTimeEtc.getDateKey
11 10 import com.piki_ds.utils.GetTextFile.getDashTable
12 11 import com.piki_ds.utils.SqlContextConf.readPartialTable
13 12 import com.piki_ds.utils.TempScoreSaveLoad
... ... @@ -40,7 +39,6 @@
40 39 }
41 40  
42 41 def followGetter(sQLContext: SQLContext, dateKey:String) = {
43   -
44 42 val fromExisting = sQLContext.read.format("json").load(recentlyUpdatedPath("/preprocess/followInfo",true,fs).getPath.toString)
45 43 val fromUpdated = getDashTable(sQLContext, "EDITOR_FOLLOW", dateKey)
46 44 val unionFrom = fromExisting.unionAll(fromUpdated)
47 45  
... ... @@ -92,10 +90,9 @@
92 90 }
93 91  
94 92 def main(args:Array[String]) {
95   - //현재시점 고정해놓기 [계산시점]Long = 1435261946000
96 93 val nowTS: Long = System.currentTimeMillis
97   - val yesterdayTuple = intoYesterdayMN(nowTS)
98   - val dateKey = yesterdayTuple._2.replaceAll("[^0-9]", "").take(8)
  94 + val dateKey = getDateKey(nowTS)
  95 +
99 96 //에디터 인기 점수 구하기
100 97 val fromDB = editorDB(sqlContext, dateKey)
101 98